DM-285: update component rules

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -80,8 +80,7 @@ public class Entity {
                 .legalBasis(e.getLegalBasis())
                 .imported(e.isImported())
                 .section(e.getSection())
-                .color(e.getColor())
+                .color(e.getColor()).positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
-                .positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -23,12 +23,12 @@ public class EntityChangeLogService {
 
 
     @Timed("redactmanager_computeChanges")
-    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> currentEntityLogEntries, int analysisNumber) {
+    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
 
         boolean hasChanges = false;
 
         var now = OffsetDateTime.now();
-        for (EntityLogEntry entityLogEntry : currentEntityLogEntries) {
+        for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
             var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
@@ -44,7 +44,7 @@ public class EntityChangeLogService {
                 }
             }
         }
-        Set<String> existingIds = currentEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
         List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
         removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
         removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -91,13 +91,13 @@ public class EntityLogCreatorService {
                                                     DictionaryVersion dictionaryVersion) {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-        List<EntityLogEntry> previousEntries = previousEntityLog.getEntityLogEntry()
+        Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
-                .stream()
+        List<EntityLogEntry> entriesFromReanalysedSections = previousEntityLog.getEntityLogEntry()
-                .filter(entry -> (entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
+                .stream().filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
                         .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
                 .toList();
-        previousEntityLog.getEntityLogEntry().removeAll(previousEntries);
+        previousEntityLog.getEntityLogEntry().removeAll(entriesFromReanalysedSections);
-        boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        boolean hasChanges = entityChangeLogService.computeChanges(entriesFromReanalysedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -106,7 +106,6 @@ public class EntityLogCreatorService {
                 false);
 
         previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
-        previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
 
         excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -4,6 +4,7 @@ import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -12,6 +13,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieSession;
 
@@ -51,25 +53,6 @@ public class ComponentCreationService {
     }
 
 
-    /**
-     * Creates a new component with the given parameters and inserts it into the kieSession.
-     *
-     * @param ruleIdentifier The rule identifier for the component.
-     * @param name The name of the component.
-     * @param value The value of the component.
-     * @param valueDescription The description of the value.
-     * @param references A collection of Entity objects that the component references.
-     */
-    public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
-
-        referencedEntities.addAll(references);
-
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
-                .references(new LinkedList<>(references))
-                .build());
-    }
-
-
     /**
      * Joins entity values, and creates a component from the result.
      *
@@ -98,6 +81,25 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Creates a new component with the given parameters and inserts it into the kieSession.
+     *
+     * @param ruleIdentifier   The rule identifier for the component.
+     * @param name             The name of the component.
+     * @param value            The value of the component.
+     * @param valueDescription The description of the value.
+     * @param references       A collection of Entity objects that the component references.
+     */
+    public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
+
+        referencedEntities.addAll(references);
+
+        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
+                .references(new LinkedList<>(references))
+                .build());
+    }
+
+
     /**
      * Joins entity values from the first section entities appear in, and creates a component from the result.
      *
@@ -210,8 +212,7 @@ public class ComponentCreationService {
 
         var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
         Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
-                .stream()
+                .stream().sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
-                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey)
                 .findFirst();
 
         if (longestSection.isEmpty()) {
@@ -327,8 +328,13 @@ public class ComponentCreationService {
         referencedEntities.add(reference);
         List<Entity> referenceList = new LinkedList<>();
         referenceList.add(reference);
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value)
+        kieSession.insert(Component.builder()
-                .valueDescription(valueDescription).references(referenceList).build());
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .name(name)
+                .value(value)
+                .valueDescription(valueDescription)
+                .references(referenceList)
+                .build());
     }
 
 
@@ -370,8 +376,25 @@ public class ComponentCreationService {
     public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
 
         String valueDescription = String.format("Convert values of type to %s joined with ', '", resultFormat);
-        String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
+        List<String> unparsedDates = new LinkedList<>();
-        create(ruleIdentifier, name, date, valueDescription, entities);
+        List<Date> dates = new LinkedList<>();
+
+        for (Entity entity : entities) {
+            String value = entity.getValue();
+            Optional<Date> optionalDate = DateConverter.parseDate(value);
+            if (optionalDate.isPresent()) {
+                dates.add(optionalDate.get());
+            } else {
+                unparsedDates.add(value);
+            }
+        }
+
+        String formattedDateStrings = Stream.concat(//
+                        dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), //
+                        unparsedDates.stream())//
+                .collect(Collectors.joining(", "));
+
+        create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
@@ -43,7 +44,11 @@ public class ComponentDroolsExecutionService {
         ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
 
         kieSession.setGlobal("componentCreationService", componentCreationService);
-        entityLog.getEntityLogEntry().stream().map(entry -> Entity.fromEntityLogEntry(entry, document)).forEach(kieSession::insert);
+        entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
+                .map(entry -> Entity.fromEntityLogEntry(entry, document))
+                .forEach(kieSession::insert);
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
@@ -55,7 +60,7 @@ public class ComponentDroolsExecutionService {
             completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
         } catch (ExecutionException e) {
             kieSession.dispose();
-            if(e.getCause() instanceof TimeoutException){
+            if (e.getCause() instanceof TimeoutException) {
                 throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
             }
             throw new RuntimeException(e);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java

@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -24,9 +25,8 @@ public class DateConverter {
             new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
 
 
-    public String convertDate(String dateAsString, String resultFormat) {
+    public Optional<Date> parseDate(String dateAsString) {
 
-        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
         Date date = null;
         for (SimpleDateFormat format : formats) {
 
@@ -38,10 +38,16 @@ public class DateConverter {
                 // ignore, try next...
             }
         }
-
         if (date == null) {
-            return dateAsString;
+            return Optional.empty();
         }
+        return Optional.of(date);
+    }
+
+
+    public String convertDate(Date date, String resultFormat) {
+
+        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
 
         return resultDateFormat.format(date);
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -51,25 +51,25 @@ rule "StudyTitle.0.0: Study Title"
     when
         $titleCandidates: List() from collect (Entity(type == "title"))
     then
-        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
+        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "");
     end
 
 
 rule "PerformingLaboratory.1.0: Performing Laboratory"
     when
-        $laboratoryName: Entity(type == "laboratory_name")
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
-        $laboratoryCountry: Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
+        $laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
-        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
+        not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
     then
-        componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!");
+        componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
     end
 
 rule "PerformingLaboratory.2.0: Performing Laboratory"
     when
-        $laboratoryName: Entity(type == "laboratory_name")
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
-        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
+        not Entity(type == "laboratory_country", containingNode == $node)
     then
-        componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!");
+        componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
     end
 
 rule "PerformingLaboratory.0.2: Performing Laboratory"
@@ -77,7 +77,7 @@ rule "PerformingLaboratory.0.2: Performing Laboratory"
     when
         not Component(name == "Performing_Laboratory")
     then
-        componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
+        componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback");
     end
 
 
@@ -85,7 +85,7 @@ rule "ReportNumber.0.0: Report number"
     when
         $reportNumberCandidates: List() from collect (Entity(type == "report_number"))
     then
-        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!");
+        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "");
     end
 
 
@@ -104,16 +104,8 @@ rule "GLPStudy.1.0: GLP Study"
     end
 
 
-rule "TestGuideline.0.0: no test guideline year found, only oecd number"
+rule "TestGuideline.0.0: create mappings"
-    when
+    salience 2
-        $guidelineNumber: Entity(type == "oecd_guideline_number")
-        not Entity(type == "oecd_guideline_year")
-    then
-        componentCreationService.create("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!");
-    end
-
-rule "TestGuideline.1.0: create mappings"
-salience 1
     when
         Entity(type == "oecd_guideline_number")
         Entity(type == "oecd_guideline_year")
@@ -149,20 +141,29 @@ salience 1
         insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
     end
 
-rule "TestGuideline.1.1: match test guidelines with mappings"
+rule "TestGuideline.0.1: match test guidelines with mappings"
+    salience 1
     when
         GuidelineMapping($year: year, $number: number, $guideline: guideline)
         $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
         $guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
     then
         componentCreationService.create(
-           "DefaultComponents.4.1",
+           "TestGuideline.0.0",
            "Test_Guidelines_1",
            $guideline,
-           "OECD Number and guideline year mapped!"
+           "OECD Number and guideline year mapped!",
+           List.of($guidelineNumber, $guidelineYear)
         );
     end
 
+rule "TestGuideline.1.0: no mapping found"
+    when
+        not Component(name == "Test_Guidelines_1")
+        $guideLine: Entity(type == "oecd_guideline")
+    then
+        componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
+    end
 
 rule "TestGuideline.2.0: Test Guideline 2"
     when