DM-285: adjust code to new EntityLog format.

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,7 +16,7 @@ val layoutParserVersion = "0.70.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "8.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.182.0"
+val persistenceServiceVersion = "2.187.0"
 
 configurations {
     all {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -10,6 +10,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -34,11 +36,11 @@ public class Entity {
 
     boolean imported;
 
+    SemanticNode containingNode;
     String section;
     float[] color;
 
     List<Position> positions;
-    int sectionNumber;
 
     String textBefore;
     String textAfter;
@@ -65,7 +67,7 @@ public class Entity {
     Set<String> importedRedactionIntersections;
 
 
-    public static Entity fromEntityLogEntry(EntityLogEntry e) {
+    public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
 
         return Entity.builder()
                 .id(e.getId())
@@ -79,8 +81,7 @@ public class Entity {
                 .imported(e.isImported())
                 .section(e.getSection())
                 .color(e.getColor())
-                .positions(e.getPositions())
+                .positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
-                .sectionNumber(e.getSectionNumber())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -249,7 +249,7 @@ public interface SemanticNode {
      * Ignores Entity with ignored == true or removed == true.
      *
      * @param types an array of strings representing the types of entities to check for
-     * @return true, if this SemanticNode has at least one Entity of any of the provided types
+     * @return true, if this SemanticNode has at least one Entity of the provided types
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
@@ -455,7 +455,7 @@ public interface SemanticNode {
      * @param y the lower left corner Y value
      * @param w width
      * @param h height
-     * @param pageNumber the pagenumber of the rectangle
+     * @param pageNumber the pageNumber of the rectangle
      * @return true if intersects, false otherwise
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
@@ -607,7 +607,7 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
 
         Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs)));
+        atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
         return bBoxPerPage;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -122,7 +122,7 @@ public class AnalyzeService {
                 dictionary.getVersion(),
                 kieWrapperEntityRules.rulesVersion());
 
-        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false),
+        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionary.getVersion(),
@@ -165,7 +165,7 @@ public class AnalyzeService {
                     entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
                             dictionaryIncrement.getDictionaryVersion(),
                             analyzeRequest.getDossierTemplateId(),
-                            false),
+                            false), document,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
@@ -206,7 +206,7 @@ public class AnalyzeService {
         return finalizeAnalysis(analyzeRequest,
                 startTime,
                 kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                entityLogChanges,
+                entityLogChanges, document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -240,7 +240,7 @@ public class AnalyzeService {
     }
 
 
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
@@ -254,7 +254,7 @@ public class AnalyzeService {
 
         log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         if (entityLogChanges.isHasChanges() || !isReanalysis) {
-            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges, dictionaryVersion);
+            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
         }
 
         log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -284,7 +284,7 @@ public class AnalyzeService {
 
 
     private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
-                                                      KieWrapper kieWrapperComponentRules,
+                                                      KieWrapper kieWrapperComponentRules, Document document,
                                                       Set<FileAttribute> addedFileAttributes,
                                                       EntityLogChanges entityLogChanges,
                                                       DictionaryVersion dictionaryVersion) {
@@ -294,7 +294,7 @@ public class AnalyzeService {
         }
 
         List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                entityLogChanges.getEntityLog(),
+                entityLogChanges.getEntityLog(), document,
                 addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -6,8 +6,9 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentValue;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.EntityReference;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
@@ -18,36 +19,35 @@ public class ComponentLogCreatorService {
 
     public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
 
-        List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component> componentLogComponents = components.stream()
+        List<ComponentLogEntry> componentLogComponents = components.stream()
                 .collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
                 .entrySet()
-                .stream()
+                .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
-                .map(entry -> new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component(entry.getKey(), entry.getValue()))
                 .toList();
         return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
     }
 
 
-    private ComponentValue buildComponentLogEntry(Component component) {
+    private ComponentLogEntryValue buildComponentLogEntry(Component component) {
 
-        return ComponentValue.builder()
+        return ComponentLogEntryValue.builder()
                 .value(component.getValue()).originalValue(component.getValue())
                 .componentRuleId(component.getMatchedRule().toString())
                 .valueDescription(component.getValueDescription())
-                .entityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
+                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
                 .build();
     }
 
 
-    private List<EntityReference> toComponentEntityReferences(List<Entity> references) {
+    private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) {
 
         return references.stream().map(this::toComponentEntityReference).toList();
     }
 
 
-    private EntityReference toComponentEntityReference(Entity entity) {
+    private ComponentLogEntityReference toComponentEntityReference(Entity entity) {
 
-        return EntityReference.builder().id(entity.getId())
+        return ComponentLogEntityReference.builder().id(entity.getId())
                 .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
                 .type(entity.getType())
                 .build();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -86,12 +86,15 @@ public class EntityLogCreatorService {
     public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
                                                     Document document,
                                                     List<ManualEntity> notFoundManualRedactionEntries,
-                                                    EntityLog previousEntityLog, Set<Integer> sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) {
+                                                    EntityLog previousEntityLog,
+                                                    Set<Integer> sectionsToReanalyseIds,
+                                                    DictionaryVersion dictionaryVersion) {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
         List<EntityLogEntry> previousEntries = previousEntityLog.getEntityLogEntry()
                 .stream()
-                .filter(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
+                .filter(entry -> (entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
+                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
                 .toList();
         previousEntityLog.getEntityLogEntry().removeAll(previousEntries);
         boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
@@ -140,8 +143,7 @@ public class EntityLogCreatorService {
     private List<EntityLogEntry> createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<EntityLogEntry> entries = new ArrayList<>();
-        document.getEntities()
+        document.getEntities().stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active)
-                .stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active)
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
         document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
         notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
@@ -182,33 +184,30 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
+    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        Set<String> referenceIds = new HashSet<>();
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
-        boolean isHint = isHint(entity.getType(), dossierTemplateId);
         return EntityLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
+                .id(image.getId())
-                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .value(image.value())
-                .legalBasis(entity.legalBasis())
+                .color(getColor(imageType, dossierTemplateId, image.applied()))
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .value(image.value())
-                .type(entity.getType())
+                .type(imageType)
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .reason(image.buildReasonWithManualChangeDescriptions())
-                .sectionNumber(sectionNumber)
+                .legalBasis(image.legalBasis())
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(entity.isDictionaryEntry())
+                .dictionaryEntry(false)
-                .textAfter(entity.getTextAfter())
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
-                .textBefore(entity.getTextBefore())
+                .containingNodeId(image.getTreeId())
-                .startOffset(entity.getTextRange().start())
+                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .endOffset(entity.getTextRange().end())
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
-                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .imageHasTransparency(image.isTransparent())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
-                .reference(referenceIds)
+                .state(buildEntryState(image))
-                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .entryType(buildEntryType(image, isHint))
-                .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity, isHint))
                 .build();
+
     }
 
 
@@ -226,8 +225,9 @@ public class EntityLogCreatorService {
                 .state(buildEntryState(manualEntity))
                 .entryType(buildEntryType(manualEntity, isHint))
                 .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
-                .sectionNumber(0)
+                .containingNodeId(Collections.emptyList())
-                .matchedRule("ManualRedaction")
+                .closestHeadline("")
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(manualEntity.isDictionaryEntry())
                 .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
                 .textAfter("")
@@ -242,29 +242,34 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        Set<String> referenceIds = new HashSet<>();
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
+        boolean isHint = isHint(entity.getType(), dossierTemplateId);
         return EntityLogEntry.builder()
-                .id(image.getId())
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
-                .value(image.value())
+                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .color(getColor(imageType, dossierTemplateId, image.applied()))
+                .legalBasis(entity.legalBasis())
-                .value(image.value())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(imageType)
+                .type(entity.getType())
-                .reason(image.buildReasonWithManualChangeDescriptions())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
-                .legalBasis(image.legalBasis())
+                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                .dictionaryEntry(false)
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .dictionaryEntry(entity.isDictionaryEntry())
-                .sectionNumber(image.getTreeId().get(0))
+                .textAfter(entity.getTextAfter())
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .textBefore(entity.getTextBefore())
-                .imageHasTransparency(image.isTransparent())
+                .startOffset(entity.getTextRange().start())
-                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
+                .endOffset(entity.getTextRange().end())
-                .state(buildEntryState(image))
+                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .entryType(buildEntryType(image, isHint))
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .reference(referenceIds)
+                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .state(buildEntryState(entity))
+                .entryType(buildEntryType(entity, isHint))
                 .build();
-
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -9,7 +9,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.OptionalInt;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -17,6 +17,9 @@ import org.kie.api.runtime.KieSession;
 
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
 import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
 
@@ -32,6 +35,14 @@ public class ComponentCreationService {
     Set<Entity> referencedEntities = new HashSet<>();
 
 
+    /**
+     * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead.
+     *
+     * @param ruleIdentifier the identifier for the rule
+     * @param name           the name of the operation
+     * @param entities       the collection of entities to search for the first value
+     * @param fallback       the value to be returned if no value is found in the collection
+     */
     public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
 
         String valueDescription = String.format("First found value or else '%s'", fallback);
@@ -40,24 +51,45 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Creates a new component with the given parameters and inserts it into the kieSession.
+     *
+     * @param ruleIdentifier The rule identifier for the component.
+     * @param name The name of the component.
+     * @param value The value of the component.
+     * @param valueDescription The description of the value.
+     * @param references A collection of Entity objects that the component references.
+     */
     public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
 
         referencedEntities.addAll(references);
 
-        kieSession.insert(Component.builder()
+        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
-                .value(value).valueDescription(valueDescription)
                 .references(new LinkedList<>(references))
                 .build());
     }
 
 
+    /**
+     * Joins entity values, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         joining(ruleIdentifier, name, entities, ", ");
     }
 
 
+    /**
+     * Joins entity values, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         String valueDescription = String.format("Joining all values with '%s'", delimiter);
@@ -66,12 +98,26 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Joins entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
     }
 
 
+    /**
+     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
@@ -81,21 +127,35 @@ public class ComponentCreationService {
 
     private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) {
 
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(Entity::getSectionNumber));
+        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
-        OptionalInt firstSection = entitiesBySection.keySet().stream().mapToInt(Integer::intValue).min();
+        Optional<SemanticNode> firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first());
         if (firstSection.isEmpty()) {
             return Collections.emptyList();
         }
-        return entitiesBySection.get(firstSection.getAsInt());
+        return entitiesBySection.get(firstSection.get());
     }
 
 
+    /**
+     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
     }
 
 
+    /**
+     * Joins entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
@@ -103,29 +163,14 @@ public class ComponentCreationService {
     }
 
 
-    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
+    /**
-
+     * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(Entity::getSectionNumber));
+     *
-        OptionalInt longestSection = entitiesBySection.entrySet()
+     * @param ruleIdentifier the identifier of the rule
-                .stream()
+     * @param name the name of the joining operation
-                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
+     * @param entities the collection of entities
-                .mapToInt(Map.Entry::getKey)
+     * @param delimiter the delimiter to use for joining the values
-                .findFirst();
+     */
-
-        if (longestSection.isEmpty()) {
-            return Collections.emptyList();
-        }
-
-        return entitiesBySection.get(longestSection.getAsInt());
-    }
-
-
-    private static int getTotalLengthOfEntities(Map.Entry<Integer, List<Entity>> entry) {
-
-        return entry.getValue().stream().mapToInt(Entity::getLength).sum();
-    }
-
-
     public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         String valueDescription = String.format("Joining all values with '%s'", delimiter);
@@ -134,6 +179,26 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Joins entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins entity values from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     */
     public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
@@ -141,6 +206,49 @@ public class ComponentCreationService {
     }
 
 
+    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
+
+        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
+                .stream()
+                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
+                .findFirst();
+
+        if (longestSection.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        return entitiesBySection.get(longestSection.get());
+    }
+
+
+    private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) {
+
+        return entry.getValue().stream().mapToInt(Entity::getLength).sum();
+    }
+
+
+    /**
+     * Joins unique entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningUniqueFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins unique entity values from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities The collection of entities to process.
+     * @param delimiter The string delimiter to join the entities.
+     */
     public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
@@ -148,40 +256,88 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Joins the names of the entities in the provided collection, separated by a comma,
+     * and creates a component from the result.
+     *
+     * @param ruleIdentifier The unique identifier of the rule being applied.
+     * @param name The name to which the joined result will be set.
+     * @param entities The collection of entities whose names will be joined.
+     */
     public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         joiningUnique(ruleIdentifier, name, entities, ", ");
     }
 
 
+    /**
+     * Computes the number of unique values in the collection of entities and creates a component with the result.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the record
+     * @param entities       the collection of entities to compute unique values from
+     */
+    public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        long count = entities.stream().map(Entity::getValue).distinct().count();
+        create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities);
+    }
+
+
+    /**
+     * Creates a component for each sentence in the collection of entities.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name The name of the entity.
+     * @param entities A collection of Entity objects.
+     */
     public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         if (entities.isEmpty()) {
             return;
         }
-
         for (Entity entity : entities) {
             BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
             iterator.setText(entity.getValue());
             int start = iterator.first();
             for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
-                create(ruleIdentifier, name, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity);
+                create(ruleIdentifier,
+                        name,
+                        entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
+                        String.format("Values of type '%s' as sentences", entity.getType()),
+                        entity);
             }
         }
     }
 
 
+    /**
+     * Creates a new component with the given rule identifier, name, value, value description, and reference.
+     * The component is built using the provided parameters and inserted into the knowledge session.
+     * The reference is added to the referencedEntities list.
+     *
+     * @param ruleIdentifier The identifier of the rule for the component.
+     * @param name The name of the component.
+     * @param value The value of the component.
+     * @param valueDescription The description of the value.
+     * @param reference The reference entity for the component.
+     */
     public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
 
         referencedEntities.add(reference);
         List<Entity> referenceList = new LinkedList<>();
         referenceList.add(reference);
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
+        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value)
-                .references(referenceList)
+                .valueDescription(valueDescription).references(referenceList).build());
-                .build());
     }
 
 
+    /**
+     * Creates components for unmapped entities.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param entities The collection of entities to create components for.
+     */
     public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
 
         entities.stream()
@@ -190,30 +346,109 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Converts entity values to the 'dd/MM/yyyy' format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name the name of the entity
+     * @param entities the collection of entities
+     */
     public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
 
         convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy");
     }
 
 
+    /**
+     * Converts entity values to provided format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name the name of the entity
+     * @param entities the collection of entities
+     * @param resultFormat the desired format for the converted dates
+     */
     public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
 
-        String valueDescription = "Convert values of type to dd/MM/yyyy joined with ', '";
+        String valueDescription = String.format("Convert values of type to %s joined with ', '", resultFormat);
         String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
         create(ruleIdentifier, name, date, valueDescription, entities);
     }
 
 
+    /**
+     * Joins values from entities that are in the same table row. If entities are not in a table cell they are added as a single component.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the entity
+     * @param entities       the collection of entities
+     */
+    public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        String types = entities.stream().map(Entity::getType).distinct().collect(Collectors.joining());
+        String valueDescription = String.format("Combine values of %s that are in same table row", types);
+        Map<Optional<Table>, List<Entity>> entitiesPerTable = entities.stream().collect(Collectors.groupingBy(this::getFirstTable));
+        entitiesPerTable.forEach((optionalTable, groupedEntities) -> {
+            if (optionalTable.isEmpty()) {
+                groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
+            }
+
+            groupedEntities.stream()
+                    .filter(entity -> !(entity.getContainingNode() instanceof TableCell))
+                    .forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
+
+            groupedEntities.stream()
+                    .filter(entity -> entity.getContainingNode() instanceof TableCell)
+                    .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
+                    .forEach((row, entitiesInSameRow) -> create(ruleIdentifier,
+                            name,
+                            entities.stream().map(Entity::getValue).collect(Collectors.joining(", ")),
+                            valueDescription,
+                            entitiesInSameRow));
+        });
+    }
+
+
+    private Optional<Table> getFirstTable(Entity entity) {
+
+        SemanticNode node = entity.getContainingNode();
+        while (!(node instanceof Table)) {
+            if (!node.hasParent()) {
+                return Optional.empty();
+            }
+            node = node.getParent();
+        }
+
+        return Optional.of((Table) node);
+    }
+
+
+    /**
+     * Creates a new component with the given rule identifier, name, value, and value description.
+     * If the component is part of a table, it also takes a list of entities that belong to the same table row.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name the name of the entity
+     * @param value the value of the component
+     * @param valueDescription the description of the value
+     */
     public void create(String ruleIdentifier, String name, String value, String valueDescription) {
 
         create(ruleIdentifier, name, value, valueDescription, Collections.emptyList());
     }
 
 
+    /**
+     * Creates a new component with the given rule identifier, name, and value.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name the name of the entity
+     * @param value the value of the component
+     */
     public void create(String ruleIdentifier, String name, String value) {
 
         kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
-                .value(value).valueDescription("")
+                .value(value)
+                .valueDescription("")
                 .references(Collections.emptyList())
                 .build());
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -47,7 +47,10 @@ public class SectionFinderService {
         Set<Integer> sectionsToReanalyse = new HashSet<>();
         for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
             if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
+                if (entry.getContainingNodeId().isEmpty()) {
+                    continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
+                }
+                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
             }
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java

@@ -0,0 +1,46 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Comparator;
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+public abstract class SemanticNodeComparators implements Comparator<SemanticNode> {
+
+    public static SemanticNodeComparators first() {
+
+        return new FirstSemanticNode();
+    }
+
+    public static class FirstSemanticNode extends SemanticNodeComparators {
+
+        @Override
+        public int compare(SemanticNode semanticNode, SemanticNode otherSemanticNode) {
+
+            List<Integer> treeId = semanticNode.getTreeId();
+            List<Integer> otherTreeId = otherSemanticNode.getTreeId();
+
+            int prefixLength = Math.min(treeId.size(), otherTreeId.size());
+            // Compare ids one by one
+            for (int i = 0; i < prefixLength; i++) {
+                int id1 = treeId.get(i);
+                int id2 = otherTreeId.get(i);
+
+                // If ids are different, return the comparison result
+                if (id1 != id2) {
+                    return Integer.compare(id1, id2);
+                }
+            }
+
+            // If all prefix ids are equal, shorter treeId is first
+            if (treeId.size() != otherTreeId.size()) {
+                return Integer.compare(treeId.size(), otherTreeId.size());
+            }
+
+            // Lists are equal
+            return 0;
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -7,8 +7,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.kie.api.runtime.rule.QueryResults;
@@ -16,11 +14,14 @@ import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -36,13 +37,13 @@ public class ComponentDroolsExecutionService {
     RedactionServiceSettings settings;
 
 
-    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, List<FileAttribute> fileAttributes) {
+    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, Document document, List<FileAttribute> fileAttributes) {
 
         KieSession kieSession = kieContainer.newKieSession();
         ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
 
         kieSession.setGlobal("componentCreationService", componentCreationService);
-        entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert);
+        entityLog.getEntityLogEntry().stream().map(entry -> Entity.fromEntityLogEntry(entry, document)).forEach(kieSession::insert);
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -113,7 +113,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow();
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
@@ -126,7 +126,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.APPLIED, asyaLyon2.getState());
 
@@ -141,13 +141,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, int sectionNumber) {
+    private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List<Integer> sectionNumber) {
 
         return redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value))
+                .filter(entry -> entry.getValue().equals(value)).filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
-                .filter(entry -> entry.getSectionNumber() == sectionNumber);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -589,11 +589,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    public void testRedactionLogAndEntityLogEquality() {
-
-    }
-
-
     @Test
     public void testRemovePublishedInformations() throws IOException {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static java.util.Map.entry;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
@@ -81,7 +80,6 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
 import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
@@ -473,7 +471,7 @@ public class RulesTest {
             assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE));
             assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
             assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
-            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
+            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId().get(0));
             assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
             assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
             assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -0,0 +1,67 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+class SemanticNodeComparatorsTest {
+
+    @Test
+    public void testFirstSemanticNode() {
+
+        var node = new Section(List.of(0, 1), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode2() {
+
+        var node = new Section(Collections.emptyList(), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode3() {
+
+        var node = new Section(List.of(1, 5, 8), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(otherNode, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode4() {
+
+        var node = new Section(List.of(1, 5, 8), null, null, null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -49,7 +49,7 @@ declare GuidelineMapping
 
 rule "StudyTitle.0.0: Study Title"
     when
-        $titleCandidates: List() from collect (Entity(type == "study_title"))
+        $titleCandidates: List() from collect (Entity(type == "title"))
     then
         componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
     end
@@ -180,38 +180,331 @@ rule "TestGuideline.2.0: Test Guideline 2"
     end
 
 
-rule "DefaultComponents.6.0: Experimental Starting Date"
+rule "StartDate.0.0: Experimental Starting Date"
     when
         $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date"))
     then
-        componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates);
+        componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates);
     end
 
 
-rule "DefaultComponents.7.0: Experimental Completion Date"
+rule "CompletionDate.0.0: Experimental Completion Date"
     when
         $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date"))
     then
-        componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates);
+        componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates);
     end
 
 
-rule "DefaultComponents.8.0: Certificate of analysis batch identification"
+rule "AnalysisCertificate.0.0: Certificate of analysis batch identification"
     when
         $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number"))
     then
-        componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers);
+        componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Batch_Number", $batchNumbers);
     end
 
 rule "StudyConclusion.0.0: Study conclusion in first found section"
     when
-        FileAttribute(label == "oecd_number", value == "425" || value == "430")
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
         $studyConclusions: List() from collect(Entity(type == "study_conclusion"))
     then
         componentCreationService.joiningUniqueFromFirstSectionOnly("Study_Conclusion.0.0", "Study_Conclusion", $studyConclusions);
     end
 
+rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
+    then
+        componentCreationService.asSentences("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations);
+    end
 
+rule "Species.0.0: First found species"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $species: List() from collect (Entity(type == "species"))
+    then
+        componentCreationService.firstOrElse("Species.0.0", "Species", $species, "");
+    end
+
+rule "Strain.0.0: First found strain"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $strain: List() from collect (Entity(type == "strain"))
+    then
+        componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, "");
+    end
+
+rule "Conclusion.0.0: Unique values of Conclusion LD50"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "ld50_value"))
+    then
+        componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions, "");
+    end
+
+rule "Conclusion0.1.0: Greater than found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater"))
+    then
+        componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater Than", "\"Greater than\" value found", $conclusions);
+    end
+
+rule "Conclusion.1.1: Greater than not found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        not Entity(type == "ld50_greater")
+    then
+        componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No \"Greater than\" value found");
+    end
+
+rule "Conclusion.2.0: Minimum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_minimal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions);
+    end
+
+rule "Conclusion.3.0: Maximum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_maximal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions);
+    end
+
+rule "Necropsy.0.0: Necropsy findings from longest section"
+    when
+        FileAttribute(label == "oecd_number", value == "402")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies);
+    end
+
+rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
+    when
+        FileAttribute(label == "oecd_number", value == "402")
+        $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)"))
+    then
+        componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
+    end
+
+rule "Necropsy.2.0: Necropsy findings as one block"
+    when
+        $oecdNumber: String() from List.of("403", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joining("Necropsy.2.0", "Necropsy_Findings", $necropsies, " ");
+    end
+
+rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block"
+    when
+        $oecdNumber: String() from List.of("403", "436")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $exposures: List() from collect (Entity(type == "4h_exposure"))
+    then
+        componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " ");
+    end
+
+rule "StudyDesign.0.0: Study design as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $studyDesigns: List() from collect (Entity(type == "study_design"))
+    then
+        componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
+    end
+
+rule "Results.0.0: Results and conclusions as joined values"
+    when
+        $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_and_conclusion"))
+    then
+        componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " ");
+    end
+
+rule "WeightBehavior.0.0: Weight change behavior as sentences"
+    when
+        FileAttribute(label == "oecd_number", value == "402")
+        $weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
+    then
+        componentCreationService.asSentences("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges);
+    end
+
+rule "MortalityStatement.0.0: Mortality statements as one block"
+    when
+        FileAttribute(label == "oecd_number", value == "402")
+        $mortalityStatements: List() from collect (Entity(type == "mortality_statement"))
+    then
+        componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " ");
+    end
+
+rule "ClinicalObservations.0.0: Clinical observations as sentences"
+    when
+        FileAttribute(label == "oecd_number", value == "403")
+        $observations: List() from collect (Entity(type == "clinical_observations"))
+    then
+        componentCreationService.asSentences("MortalityStatement.0.0", "Clinical_Observations", $observations);
+    end
+
+rule "BodyWeight.0.0: Bodyweight changes as sentences"
+    when
+        FileAttribute(label == "oecd_number", value == "403")
+        $weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
+    then
+        componentCreationService.asSentences("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges);
+    end
+
+rule "Detailing.0.0: Detailing of reported changes as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $detailings: List() from collect (Entity(type == "detailing"))
+    then
+        componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " ");
+    end
+
+rule "Sex.0.0: Male sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $males: List(!isEmpty) from collect (Entity(type == "sex", (value == "male" || value == "males")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males);
+    end
+
+rule "Sex.1.0: Female sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $females: List(!isEmpty) from collect (Entity(type == "sex", (value == "female" || value == "females")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females);
+    end
+
+rule "NumberOfAnimals.0.0: Number of animals found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $numberOfAnimals: Entity(type == "number_of_animals")
+    then
+        componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals);
+    end
+
+rule "NumberOfAnimals.1.0: Count unique occurences of animals"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        not Entity(type == "number_of_animals")
+        $animals: List() from collect (Entity(type == "animal_number"))
+    then
+        componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals);
+    end
+
+rule "ClinicalSigns.0.0: Clinical signs as sentences"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
+    then
+        componentCreationService.asSentences("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns);
+    end
+
+rule "DoseMortality.0.0: Dose mortality as sentences"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
+    then
+        componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities);
+    end
+
+rule "Mortality.0.0: Mortality as one block"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "mortality"))
+    then
+        componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " ");
+    end
+
+rule "Dosages.0.0: First found value of Dosages"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "mortality"))
+    then
+        componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, "");
+    end
+
+rule "PrelimResults.0.0: Preliminary test results as sentences"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "preliminary_test_results"))
+    then
+        componentCreationService.asSentences("PrelimResults.0.0", "Preliminary_Test_Results", $results);
+    end
+
+rule "TestResults.0.0: Test results as one block"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "test_results"))
+    then
+        componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " ");
+    end
+
+rule "PositiveControl.0.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "positive_control"))
+    then
+        componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " ");
+    end
+
+rule "MainResults.0.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_(main_study)"))
+    then
+        componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " ");
+    end
+
+rule "UsedApproach.0.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        $results: List(!isEmpty()) from collect (Entity(type == "approach_used"))
+    then
+        componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results);
+    end
+
+rule "UsedApproach.1.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        not Entity(type == "approach_used")
+    then
+        componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
+    end
+
+/*
 rule "DefaultComponents.999.0: Create components for all unmapped entities."
     salience -999
     when
@@ -220,7 +513,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities."
         componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
     end
 
-
+*/
 
 //------------------------------------ Component merging rules ------------------------------------