DM-285: sort dates in ComponentCreationService

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -1,7 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
-import java.util.stream.Collectors;
+import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
@@ -12,6 +14,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
 
 @Service
@@ -19,8 +22,12 @@ public class ComponentLogCreatorService {
 
     public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
 
-        List<ComponentLogEntry> componentLogComponents = components.stream()
+        Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
-                .collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
+        components.stream().sorted(ComponentComparator.first()).forEach(component -> {
+            ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
+            map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
+        });
+        List<ComponentLogEntry> componentLogComponents = map
                 .entrySet()
                 .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
                 .toList();
@@ -34,7 +41,7 @@ public class ComponentLogCreatorService {
                 .value(component.getValue()).originalValue(component.getValue())
                 .componentRuleId(component.getMatchedRule().toString())
                 .valueDescription(component.getValueDescription())
-                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList()))
+                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList()))
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+
+public class ComponentComparator implements Comparator<Component> {
+
+    public static ComponentComparator first() {
+
+        return new ComponentComparator();
+    }
+
+
+    @Override
+    public int compare(Component component1, Component component2) {
+
+        var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first());
+        var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first());
+        if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) {
+            return 0;
+        }
+        if (firstEntity1.isEmpty() && firstEntity2.isPresent()) {
+            return -1;
+        }
+        if (firstEntity1.isPresent() && firstEntity2.isEmpty()) {
+            return 1;
+        }
+
+        return new EntityComparators.FirstEntity().compare(firstEntity1.get(), firstEntity2.get());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -37,19 +37,20 @@ public class ComponentCreationService {
     Set<Entity> referencedEntities = new HashSet<>();
 
 
-    /**
+    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
-     * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead.
-     *
-     * @param ruleIdentifier the identifier for the rule
-     * @param name           the name of the operation
-     * @param entities       the collection of entities to search for the first value
-     * @param fallback       the value to be returned if no value is found in the collection
-     */
-    public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
 
-        String valueDescription = String.format("First found value or else '%s'", fallback);
+        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
-        String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
+        Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
-        create(ruleIdentifier, name, value, valueDescription, entities);
+                .stream()
+                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
+                .map(Map.Entry::getKey)
+                .findFirst();
+
+        if (longestSection.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        return entitiesBySection.get(longestSection.get());
     }
 
 
@@ -67,20 +68,27 @@ public class ComponentCreationService {
 
 
     /**
-     * Joins entity values, and creates a component from the result.
+     * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead.
      *
-     * @param ruleIdentifier The identifier of the rule.
+     * @param ruleIdentifier the identifier for the rule
-     * @param name           The name of the entity.
+     * @param name           the name of the operation
-     * @param entities       The collection of entities to process.
+     * @param entities       the collection of entities to search for the first value
+     * @param fallback       the value to be returned if no value is found in the collection
      */
-    public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+    public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
 
-        String valueDescription = String.format("Joining all values with '%s'", delimiter);
+        String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback);
-        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
+        String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback);
         create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
 
+    private static String joinTypes(Collection<Entity> entities) {
+
+        return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", "));
+    }
+
+
     /**
      * Creates a new component with the given parameters and inserts it into the kieSession.
      *
@@ -166,17 +174,16 @@ public class ComponentCreationService {
 
 
     /**
-     * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
+     * Joins entity values, and creates a component from the result.
      *
-     * @param ruleIdentifier the identifier of the rule
+     * @param ruleIdentifier The identifier of the rule.
-     * @param name           the name of the joining operation
+     * @param name           The name of the entity.
-     * @param entities       the collection of entities
+     * @param entities       The collection of entities to process.
-     * @param delimiter      the delimiter to use for joining the values
      */
-    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+    public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
-        String valueDescription = String.format("Joining all values with '%s'", delimiter);
+        String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter);
-        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
+        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter));
         create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
@@ -208,18 +215,19 @@ public class ComponentCreationService {
     }
 
 
-    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
+    /**
+     * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the joining operation
+     * @param entities       the collection of entities
+     * @param delimiter      the delimiter to use for joining the values
+     */
+    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter);
-        Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
+        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
-                .stream().sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
+        create(ruleIdentifier, name, value, valueDescription, entities);
-                .findFirst();
-
-        if (longestSection.isEmpty()) {
-            return Collections.emptyList();
-        }
-
-        return entitiesBySection.get(longestSection.get());
     }
 
 
@@ -297,7 +305,7 @@ public class ComponentCreationService {
         if (entities.isEmpty()) {
             return;
         }
-        for (Entity entity : entities) {
+        entities.stream().sorted(EntityComparators.first()).forEach(entity -> {
             BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
             iterator.setText(entity.getValue());
             int start = iterator.first();
@@ -308,7 +316,7 @@ public class ComponentCreationService {
                         String.format("Values of type '%s' as sentences", entity.getType()),
                         entity);
             }
-        }
+        });
     }
 
 
@@ -346,8 +354,7 @@ public class ComponentCreationService {
      */
     public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
 
-        entities.stream()
+        entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first())
-                .filter(entity -> !referencedEntities.contains(entity))
                 .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
     }
 
@@ -375,7 +382,7 @@ public class ComponentCreationService {
      */
     public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
 
-        String valueDescription = String.format("Convert values of type to %s joined with ', '", resultFormat);
+        String valueDescription = String.format("Convert values of type '%s' to %s joined with ', '", joinTypes(entities), resultFormat);
         List<String> unparsedDates = new LinkedList<>();
         List<Date> dates = new LinkedList<>();
 
@@ -407,10 +414,9 @@ public class ComponentCreationService {
      */
     public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        String types = entities.stream().map(Entity::getType).distinct().collect(Collectors.joining());
+        String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", "));
         String valueDescription = String.format("Combine values of %s that are in same table row", types);
-        Map<Optional<Table>, List<Entity>> entitiesPerTable = entities.stream().collect(Collectors.groupingBy(this::getFirstTable));
+        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
-        entitiesPerTable.forEach((optionalTable, groupedEntities) -> {
             if (optionalTable.isEmpty()) {
                 groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
             }
@@ -422,9 +428,13 @@ public class ComponentCreationService {
             groupedEntities.stream()
                     .filter(entity -> entity.getContainingNode() instanceof TableCell)
                     .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
-                    .forEach((row, entitiesInSameRow) -> create(ruleIdentifier,
+                    .entrySet()
+                    .stream()
+                    .sorted(Comparator.comparingInt(Map.Entry::getKey))
+                    .map(Map.Entry::getValue)
+                    .forEach(entitiesInSameRow -> create(ruleIdentifier,
                             name,
-                            entities.stream().map(Entity::getValue).collect(Collectors.joining(", ")),
+                            entitiesInSameRow.stream().sorted(Comparator.comparing(Entity::getType).reversed()).map(Entity::getValue).collect(Collectors.joining(", ")),
                             valueDescription,
                             entitiesInSameRow));
         });

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java

@@ -6,7 +6,12 @@ import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 
 public abstract class EntityComparators implements Comparator<Entity> {
 
-    private static class LongestEntity implements Comparator<Entity> {
+    public static Comparator<Entity> first() {
+
+        return new FirstEntity();
+    }
+
+    public static class LongestEntity implements Comparator<Entity> {
 
         @Override
         public int compare(Entity Entity, Entity otherEntity) {
@@ -16,26 +21,20 @@ public abstract class EntityComparators implements Comparator<Entity> {
 
     }
 
-    private static class FirstEntity implements Comparator<Entity> {
-
-        @Override
-        public int compare(Entity Entity, Entity otherEntity) {
-
-            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
-        }
-
-    }
-
 
     public static Comparator<Entity> length() {
 
         return new LongestEntity();
     }
 
+    public static class FirstEntity implements Comparator<Entity> {
 
-    public static Comparator<Entity> start() {
+        @Override
+        public int compare(Entity Entity, Entity otherEntity) {
+
+            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
+        }
 
-        return new FirstEntity();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -21,6 +21,7 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
@@ -70,7 +71,7 @@ public class ComponentDroolsExecutionService {
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        List<Component> components = getComponents(kieSession);
+        List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList();
         kieSession.dispose();
         return components;
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -48,7 +48,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 //    @Disabled
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/_000008810-0.2.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/A8591B/15-Curacron_ToxicidadeAgudaOral.pdf");
 
 //        AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf",
 //                "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -78,6 +78,7 @@ import lombok.SneakyThrows;
 public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl");
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.wildfly.common.Assert.assertFalse;
 
 import java.awt.geom.Rectangle2D;
+import java.io.FileOutputStream;
 import java.time.OffsetDateTime;
 import java.util.Collection;
 import java.util.Comparator;
@@ -27,22 +28,30 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+
+import lombok.SneakyThrows;
 
 @Import(ManualChangesIntegrationTest.TestConfiguration.class)
 public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
@@ -249,6 +258,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.removed());
     }
 
+
     private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
 
         if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+
+class EntityComparatorsTest {
+
+    @Test
+    public void testFirstEntity() {
+
+        Entity entity1 = Entity.builder().startOffset(0).build();
+        Entity entity2 = Entity.builder().startOffset(2).build();
+        Entity entity3 = Entity.builder().startOffset(20).build();
+        assertEquals(entity1, Stream.of(entity2, entity3, entity1).min(EntityComparators.first()).orElseThrow());
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -47,7 +47,7 @@ declare GuidelineMapping
 
 //------------------------------------ Default Components rules ------------------------------------
 
-rule "StudyTitle.0.0: Study Title"
+rule "StudyTitle.0.0: First Title found"
     when
         $titleCandidates: List() from collect (Entity(type == "title"))
     then
@@ -55,7 +55,7 @@ rule "StudyTitle.0.0: Study Title"
     end
 
 
-rule "PerformingLaboratory.1.0: Performing Laboratory"
+rule "PerformingLaboratory.1.0: Performing Laboratory name and country found in same section"
     when
         $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
         $laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
@@ -64,7 +64,7 @@ rule "PerformingLaboratory.1.0: Performing Laboratory"
         componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
     end
 
-rule "PerformingLaboratory.2.0: Performing Laboratory"
+rule "PerformingLaboratory.2.0: Performing Laboratory name but no country found in same section"
     when
         $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
         not Entity(type == "laboratory_country", containingNode == $node)
@@ -72,7 +72,7 @@ rule "PerformingLaboratory.2.0: Performing Laboratory"
         componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
     end
 
-rule "PerformingLaboratory.0.2: Performing Laboratory"
+rule "PerformingLaboratory.0.2: Performing Laboratory not found"
     salience -1
     when
         not Component(name == "Performing_Laboratory")
@@ -81,30 +81,30 @@ rule "PerformingLaboratory.0.2: Performing Laboratory"
     end
 
 
-rule "ReportNumber.0.0: Report number"
+rule "ReportNumber.0.0: First Report number found"
     when
         $reportNumberCandidates: List() from collect (Entity(type == "report_number"))
     then
-        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "");
+        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_Number", $reportNumberCandidates, "");
     end
 
 
-rule "GLPStudy.0.0: GLP Study"
+rule "GLPStudy.0.0: GLP Study found"
     when
         $glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study"))
     then
-        componentCreationService.create("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudyList);
+        componentCreationService.create("GLPStudy.0.0", "GLP_Study", "Yes", "Yes if present, No if not", $glpStudyList);
     end
 
-rule "GLPStudy.1.0: GLP Study"
+rule "GLPStudy.1.0: GLP Study not found"
     when
         not Entity(type == "glp_study")
     then
-        componentCreationService.create("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not");
+        componentCreationService.create("GLPStudy.1.0", "GLP_Study", "No", "Yes if present, No if not");
     end
 
 
-rule "TestGuideline.0.0: create mappings"
+rule "TestGuideline.0.0: create OECD number and year guideline mappings"
     salience 2
     when
         Entity(type == "oecd_guideline_number")
@@ -141,7 +141,7 @@ rule "TestGuideline.0.0: create mappings"
         insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
     end
 
-rule "TestGuideline.0.1: match test guidelines with mappings"
+rule "TestGuideline.0.1: match OECD number and year with guideline mappings"
     salience 1
     when
         GuidelineMapping($year: year, $number: number, $guideline: guideline)
@@ -157,7 +157,7 @@ rule "TestGuideline.0.1: match test guidelines with mappings"
         );
     end
 
-rule "TestGuideline.1.0: no mapping found"
+rule "TestGuideline.1.0: no guideline mapping found"
     when
         not Component(name == "Test_Guidelines_1")
         $guideLine: Entity(type == "oecd_guideline")
@@ -165,23 +165,15 @@ rule "TestGuideline.1.0: no mapping found"
         componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
     end
 
-rule "TestGuideline.2.0: Test Guideline 2"
+rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines"
     when
-        $epaGuideLines: List() from collect (Entity(type == "epa_guideline"))
+        $guidelines: List() from collect (Entity(type == "epa_guideline" || type ==  "ec_guideline"))
-        $ecGuideLines: List() from collect (Entity(type == "ec_guideline"))
     then
-        componentCreationService.joining("TestGuideline.2.0",
+        componentCreationService.joining("TestGuideline.2.0", "Test_Guidelines_2", $guidelines);
-            "Test_Guideline_2",
-            Stream.of(
-              $epaGuideLines.stream(),
-              $ecGuideLines.stream())
-          .flatMap(a -> a)
-          .toList()
-        );
     end
 
 
-rule "StartDate.0.0: Experimental Starting Date"
+rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
     when
         $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date"))
     then
@@ -189,7 +181,7 @@ rule "StartDate.0.0: Experimental Starting Date"
     end
 
 
-rule "CompletionDate.0.0: Experimental Completion Date"
+rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
     when
         $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date"))
     then
@@ -197,26 +189,26 @@ rule "CompletionDate.0.0: Experimental Completion Date"
     end
 
 
-rule "AnalysisCertificate.0.0: Certificate of analysis batch identification"
+rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification"
     when
         $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number"))
     then
-        componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Batch_Number", $batchNumbers);
+        componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers);
     end
 
 rule "StudyConclusion.0.0: Study conclusion in first found section"
     when
         $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $studyConclusions: List() from collect(Entity(type == "study_conclusion"))
     then
-        componentCreationService.joiningUniqueFromFirstSectionOnly("Study_Conclusion.0.0", "Study_Conclusion", $studyConclusions);
+        componentCreationService.joiningFromFirstSectionOnly("StudyConclusion.0.0", "Study_Conclusion", $studyConclusions, " ");
     end
 
 rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
     when
         $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
     then
         componentCreationService.asSentences("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations);
@@ -225,7 +217,7 @@ rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
 rule "Species.0.0: First found species"
     when
         $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $species: List() from collect (Entity(type == "species"))
     then
         componentCreationService.firstOrElse("Species.0.0", "Species", $species, "");
@@ -234,7 +226,7 @@ rule "Species.0.0: First found species"
 rule "Strain.0.0: First found strain"
     when
         $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $strain: List() from collect (Entity(type == "strain"))
     then
         componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, "");
@@ -243,34 +235,34 @@ rule "Strain.0.0: First found strain"
 rule "Conclusion.0.0: Unique values of Conclusion LD50"
     when
         $oecdNumber: String() from List.of("402", "403", "425", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $conclusions: List() from collect (Entity(type == "ld50_value"))
     then
-        componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions, "");
+        componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions);
     end
 
 rule "Conclusion0.1.0: Greater than found"
     when
         $oecdNumber: String() from List.of("402", "403", "425", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater"))
     then
-        componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater Than", "\"Greater than\" value found", $conclusions);
+        componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater than", "Entity of type 'ld50_greater' found", $conclusions);
     end
 
 rule "Conclusion.1.1: Greater than not found"
     when
         $oecdNumber: String() from List.of("402", "403", "425", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         not Entity(type == "ld50_greater")
     then
-        componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No \"Greater than\" value found");
+        componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No entity of type 'ld50_greater' found");
     end
 
 rule "Conclusion.2.0: Minimum confidence as unique values"
     when
         $oecdNumber: String() from List.of("402", "403", "425", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $conclusions: List() from collect (Entity(type == "confidence_minimal"))
     then
         componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions);
@@ -279,7 +271,7 @@ rule "Conclusion.2.0: Minimum confidence as unique values"
 rule "Conclusion.3.0: Maximum confidence as unique values"
     when
         $oecdNumber: String() from List.of("402", "403", "425", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $conclusions: List() from collect (Entity(type == "confidence_maximal"))
     then
         componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions);
@@ -287,15 +279,23 @@ rule "Conclusion.3.0: Maximum confidence as unique values"
 
 rule "Necropsy.0.0: Necropsy findings from longest section"
     when
-        FileAttribute(label == "oecd_number", value == "402")
+        FileAttribute(label == "OECD Number", value == "402")
         $necropsies: List() from collect (Entity(type == "necropsy_findings"))
     then
-        componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies);
+        componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " ");
+    end
+
+rule "Necropsy.0.1: Necropsy findings from longest section"
+    when
+        FileAttribute(label == "OECD Number", value == "403" || value == "436")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.asSentences("Necropsy.0.0", "Necropsy_Findings", $necropsies);
     end
 
 rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
     when
-        FileAttribute(label == "oecd_number", value == "402")
+        FileAttribute(label == "OECD Number", value == "402")
         $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)"))
     then
         componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
@@ -304,7 +304,7 @@ rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
 rule "Necropsy.2.0: Necropsy findings as one block"
     when
         $oecdNumber: String() from List.of("403", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $necropsies: List() from collect (Entity(type == "necropsy_findings"))
     then
         componentCreationService.joining("Necropsy.2.0", "Necropsy_Findings", $necropsies, " ");
@@ -313,7 +313,7 @@ rule "Necropsy.2.0: Necropsy findings as one block"
 rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block"
     when
         $oecdNumber: String() from List.of("403", "436")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $exposures: List() from collect (Entity(type == "4h_exposure"))
     then
         componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " ");
@@ -322,7 +322,7 @@ rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block"
 rule "StudyDesign.0.0: Study design as one block"
     when
         $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $studyDesigns: List() from collect (Entity(type == "study_design"))
     then
         componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
@@ -331,7 +331,7 @@ rule "StudyDesign.0.0: Study design as one block"
 rule "Results.0.0: Results and conclusions as joined values"
     when
         $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List() from collect (Entity(type == "results_and_conclusion"))
     then
         componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " ");
@@ -339,7 +339,7 @@ rule "Results.0.0: Results and conclusions as joined values"
 
 rule "WeightBehavior.0.0: Weight change behavior as sentences"
     when
-        FileAttribute(label == "oecd_number", value == "402")
+        FileAttribute(label == "OECD Number", value == "402")
         $weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
     then
         componentCreationService.asSentences("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges);
@@ -347,7 +347,7 @@ rule "WeightBehavior.0.0: Weight change behavior as sentences"
 
 rule "MortalityStatement.0.0: Mortality statements as one block"
     when
-        FileAttribute(label == "oecd_number", value == "402")
+        FileAttribute(label == "OECD Number", value == "402")
         $mortalityStatements: List() from collect (Entity(type == "mortality_statement"))
     then
         componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " ");
@@ -355,7 +355,7 @@ rule "MortalityStatement.0.0: Mortality statements as one block"
 
 rule "ClinicalObservations.0.0: Clinical observations as sentences"
     when
-        FileAttribute(label == "oecd_number", value == "403")
+        FileAttribute(label == "OECD Number", value == "403")
         $observations: List() from collect (Entity(type == "clinical_observations"))
     then
         componentCreationService.asSentences("MortalityStatement.0.0", "Clinical_Observations", $observations);
@@ -363,7 +363,7 @@ rule "ClinicalObservations.0.0: Clinical observations as sentences"
 
 rule "BodyWeight.0.0: Bodyweight changes as sentences"
     when
-        FileAttribute(label == "oecd_number", value == "403")
+        FileAttribute(label == "OECD Number", value == "403")
         $weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
     then
         componentCreationService.asSentences("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges);
@@ -372,7 +372,7 @@ rule "BodyWeight.0.0: Bodyweight changes as sentences"
 rule "Detailing.0.0: Detailing of reported changes as one block"
     when
         $oecdNumber: String() from List.of("404", "405")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $detailings: List() from collect (Entity(type == "detailing"))
     then
         componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " ");
@@ -381,8 +381,8 @@ rule "Detailing.0.0: Detailing of reported changes as one block"
 rule "Sex.0.0: Male sex found"
     when
         $oecdNumber: String() from List.of("405", "429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
-        $males: List(!isEmpty) from collect (Entity(type == "sex", (value == "male" || value == "males")))
+        $males: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "male" || value.toLowerCase() == "males")))
     then
         componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males);
     end
@@ -390,8 +390,8 @@ rule "Sex.0.0: Male sex found"
 rule "Sex.1.0: Female sex found"
     when
         $oecdNumber: String() from List.of("405", "429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
-        $females: List(!isEmpty) from collect (Entity(type == "sex", (value == "female" || value == "females")))
+        $females: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "female" || value.toLowerCase() == "females")))
     then
         componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females);
     end
@@ -399,7 +399,7 @@ rule "Sex.1.0: Female sex found"
 rule "NumberOfAnimals.0.0: Number of animals found"
     when
         $oecdNumber: String() from List.of("405", "429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $numberOfAnimals: Entity(type == "number_of_animals")
     then
         componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals);
@@ -408,7 +408,7 @@ rule "NumberOfAnimals.0.0: Number of animals found"
 rule "NumberOfAnimals.1.0: Count unique occurences of animals"
     when
         $oecdNumber: String() from List.of("405", "429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         not Entity(type == "number_of_animals")
         $animals: List() from collect (Entity(type == "animal_number"))
     then
@@ -418,16 +418,16 @@ rule "NumberOfAnimals.1.0: Count unique occurences of animals"
 rule "ClinicalSigns.0.0: Clinical signs as sentences"
     when
         $oecdNumber: String() from List.of("425")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
     then
         componentCreationService.asSentences("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns);
     end
 
-rule "DoseMortality.0.0: Dose mortality as sentences"
+rule "DoseMortality.0.0: Dose mortality joined with dose from same table row"
     when
         $oecdNumber: String() from List.of("425")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
     then
         componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities);
@@ -436,7 +436,7 @@ rule "DoseMortality.0.0: Dose mortality as sentences"
 rule "Mortality.0.0: Mortality as one block"
     when
         $oecdNumber: String() from List.of("425")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $mortalities: List() from collect (Entity(type == "mortality"))
     then
         componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " ");
@@ -445,8 +445,8 @@ rule "Mortality.0.0: Mortality as one block"
 rule "Dosages.0.0: First found value of Dosages"
     when
         $oecdNumber: String() from List.of("425")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
-        $mortalities: List() from collect (Entity(type == "mortality"))
+        $mortalities: List() from collect (Entity(type == "dosages"))
     then
         componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, "");
     end
@@ -454,7 +454,7 @@ rule "Dosages.0.0: First found value of Dosages"
 rule "PrelimResults.0.0: Preliminary test results as sentences"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List() from collect (Entity(type == "preliminary_test_results"))
     then
         componentCreationService.asSentences("PrelimResults.0.0", "Preliminary_Test_Results", $results);
@@ -463,7 +463,7 @@ rule "PrelimResults.0.0: Preliminary test results as sentences"
 rule "TestResults.0.0: Test results as one block"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List() from collect (Entity(type == "test_results"))
     then
         componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " ");
@@ -472,34 +472,34 @@ rule "TestResults.0.0: Test results as one block"
 rule "PositiveControl.0.0: Was the definitive study conducted with positive control"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List() from collect (Entity(type == "positive_control"))
     then
         componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " ");
     end
 
-rule "MainResults.0.0: Was the definitive study conducted with positive control"
+rule "MainResults.0.0: Results from main study as one block"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List() from collect (Entity(type == "results_(main_study)"))
     then
         componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " ");
     end
 
-rule "UsedApproach.0.0: Was the definitive study conducted with positive control"
+rule "UsedApproach.0.0: Used approach found and mapped to 'Group'"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         $results: List(!isEmpty()) from collect (Entity(type == "approach_used"))
     then
         componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results);
     end
 
-rule "UsedApproach.1.0: Was the definitive study conducted with positive control"
+rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
     when
         $oecdNumber: String() from List.of("429")
-        FileAttribute(label == "oecd_number", value == $oecdNumber)
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
         not Entity(type == "approach_used")
     then
         componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
@@ -513,11 +513,10 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities."
     then
         componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
     end
-
 */
 
 //------------------------------------ Component merging rules ------------------------------------
-
+/*
 rule "X.0.0: merge duplicate component references"
     when
         $first: Component()
@@ -526,3 +525,4 @@ rule "X.0.0: merge duplicate component references"
         $first.getReferences().addAll($duplicate.getReferences());
         retract($duplicate);
     end
+*/