DM-285: Entity to component mappings with new ruleset

* refactoring structure
* added custom syntax validation, such that rule files may not be corrupted
* added component drools preparations
* deprecated redactionLog
* integrated entityLog
* integrated new getRules with ruleFileType
* renamed service.graph package -> service.document
* fixed tests
* update layout parser version
* initial component drools file - wip

Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.25.0"
+val layoutParserVersion = "0.70.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "8.43.0.Final"
-val pdfBoxVersion = "3.0.0-alpha2"
+val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "DM-285-3"
+val persistenceServiceVersion = "DM-285-5"
 
 configurations {
     all {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -5,7 +5,8 @@ import java.util.PriorityQueue;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
@@ -20,7 +21,7 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ManualEntity implements Entity {
+public class ManualEntity implements IEntity {
 
     // The id must be mapped into a TextEntity as is for comments to work correctly
     String id;
@@ -88,4 +89,18 @@ public class ManualEntity implements Entity {
                 .build();
     }
 
+
+    @Override
+    public TextRange getTextRange() {
+
+        return new TextRange(-1, -1);
+    }
+
+
+    @Override
+    public String type() {
+
+        return getManualOverwrite().getType().orElse(type);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java

@@ -0,0 +1,41 @@
+package com.iqser.red.service.redaction.v1.server.model.component;
+
+import java.util.Collection;
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class Component {
+
+    RuleIdentifier matchedRule;
+    String category;
+    String value;
+    String transformation;
+
+    List<Entity> references;
+
+
+    public boolean addReference(Entity entity) {
+
+        return references.add(entity);
+    }
+
+
+    public boolean addReferences(Collection<Entity> entities) {
+
+        return references.addAll(entities);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -0,0 +1,106 @@
+package com.iqser.red.service.redaction.v1.server.model.component;
+
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class Entity {
+
+    String id;
+    String type;
+    EntryType entryType;
+    EntryState state;
+    String value;
+    String reason;
+    String matchedRule;
+    boolean rectangle;
+    String legalBasis;
+
+    boolean imported;
+
+    String section;
+    float[] color;
+
+    List<Position> positions;
+    int sectionNumber;
+
+    String textBefore;
+    String textAfter;
+
+    int startOffset;
+    int endOffset;
+    int length;
+
+    boolean imageHasTransparency;
+
+    boolean isDictionaryEntry;
+    boolean isDossierDictionaryEntry;
+
+    boolean excluded;
+
+    String sourceId;
+
+    List<Change> changes;
+
+    List<ManualChange> manualChanges;
+
+    Set<Engine> engines;
+
+    Set<String> reference;
+
+    Set<String> importedRedactionIntersections;
+
+
+    public static Entity fromEntityLogEntry(EntityLogEntry e) {
+
+        return Entity.builder()
+                .id(e.getId())
+                .type(e.getType())
+                .entryType(e.getEntryType())
+                .state(e.getState())
+                .value(e.getValue())
+                .reason(e.getReason())
+                .matchedRule(e.getMatchedRule())
+                .rectangle(e.isRectangle())
+                .legalBasis(e.getLegalBasis())
+                .imported(e.isImported())
+                .section(e.getSection())
+                .color(e.getColor())
+                .positions(e.getPositions())
+                .sectionNumber(e.getSectionNumber())
+                .textBefore(e.getTextBefore())
+                .textAfter(e.getTextAfter())
+                .startOffset(e.getStartOffset())
+                .endOffset(e.getEndOffset())
+                .length(e.getValue().length())
+                .imageHasTransparency(e.isImageHasTransparency())
+                .isDictionaryEntry(e.isDictionaryEntry())
+                .isDossierDictionaryEntry(e.isDossierDictionaryEntry())
+                .excluded(e.isExcluded())
+                .sourceId(e.getSourceId())
+                .changes(e.getChanges())
+                .manualChanges(e.getManualChanges())
+                .engines(e.getEngines())
+                .reference(e.getReference())
+                .importedRedactionIntersections(e.getImportedRedactionIntersections())
+                .build();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java

@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
-
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Component {
-
-    String value;
-    String originalValue;
-    String transformation;
-
-    List<Entity> references;
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -5,11 +5,12 @@ import java.util.HashSet;
 import java.util.PriorityQueue;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
 
 import lombok.NonNull;
 
-public interface Entity {
+public interface IEntity {
 
     PriorityQueue<MatchedRule> getMatchedRuleList();
 
@@ -17,6 +18,27 @@ public interface Entity {
     ManualChangeOverwrite getManualOverwrite();
 
 
+    String getValue();
+
+
+    TextRange getTextRange();
+
+
+    String type();
+
+
+    default int length() {
+
+        return value().length();
+    }
+
+
+    default String value() {
+
+        return getManualOverwrite().getValue().orElse(getValue());
+    }
+
+
     // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
     default boolean applied() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
-public class TextEntity implements Entity {
+public class TextEntity implements IEntity {
 
     // primary key
     @EqualsAndHashCode.Include
@@ -201,5 +201,10 @@ public class TextEntity implements Entity {
     }
 
 
+    @Override
+    public String type() {
+
+        return getManualOverwrite().getType().orElse(type);
+    }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -5,12 +5,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -30,7 +32,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class Image implements GenericSemanticNode, Entity {
+public class Image implements GenericSemanticNode, IEntity {
 
     List<Integer> treeId;
     String id;
@@ -77,6 +79,20 @@ public class Image implements GenericSemanticNode, Entity {
     }
 
 
+    @Override
+    public TextRange getTextRange() {
+
+        return GenericSemanticNode.super.getTextRange();
+    }
+
+
+    @Override
+    public String type() {
+
+        return getManualOverwrite().getType().orElse(imageType.toString());
+    }
+
+
     @Override
     public String toString() {
 
@@ -92,4 +108,23 @@ public class Image implements GenericSemanticNode, Entity {
         return bBoxPerPage;
     }
 
+
+    @Override
+    public String getValue() {
+
+        return NodeType.IMAGE + ":" + camelCase(imageType.toString());
+    }
+
+
+    private String camelCase(String name) {
+
+        return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
+    }
+
+
+    public int length() {
+
+        return 0;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java

@@ -16,6 +16,6 @@ public enum NodeType {
 
     public String toString() {
 
-        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH);
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
@@ -35,15 +36,16 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
+import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -71,6 +73,7 @@ public class AnalyzeService {
     ComponentLogCreatorService componentLogCreatorService;
     RedactionStorageService redactionStorageService;
     RedactionChangeLogService redactionChangeLogService;
+    EntityChangeLogService entityChangeLogService;
     LegalBasisClient legalBasisClient;
     RedactionServiceSettings redactionServiceSettings;
     ImportedRedactionService importedRedactionService;
@@ -114,27 +117,14 @@ public class AnalyzeService {
                 nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        ComponentLog componentLog = new ComponentLog();
-        if (kieWrapperComponentRules.isPresent()) {
-            List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
-                    allFileAttributes,
-                    notFoundManualRedactionEntries,
-                    document.streamAllImages().toList());
-
-            componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
-            log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        }
-
         RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
         EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
+                startTime, kieWrapperComponentRules, entityLog,
-                entityLog, componentLog,
                 redactionLog,
                 document.getNumberOfPages(),
-                dictionary.getVersion(),
+                dictionary.getVersion(), false, new HashSet<>(allFileAttributes));
-                false, new HashSet<>(allFileAttributes));
     }
 
 
@@ -184,16 +174,6 @@ public class AnalyzeService {
         log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         ComponentLog previousComponentLog = new ComponentLog();
 
-        KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
-        if (kieWrapperComponentRules.isPresent()) {
-            log.info("Updated component rules to version {} for file {} in dossier {}",
-                    kieWrapperComponentRules.rulesVersion(),
-                    analyzeRequest.getFileId(),
-                    analyzeRequest.getDossierId());
-            previousComponentLog = redactionStorageService.getComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-            log.info("Loaded previous component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        }
-
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -203,30 +183,28 @@ public class AnalyzeService {
         }
 
         DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                new DictionaryVersion(previousRedactionLog.getDictionaryVersion(), previousRedactionLog.getDossierDictionaryVersion()),
+                new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
                 analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement);
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
         log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest,
-                    startTime,
+                    startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog,
-                    previousEntityLog, previousComponentLog,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
                     true,
                     Collections.emptySet());
         }
+        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
+        log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
-        log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
         List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
@@ -244,23 +222,11 @@ public class AnalyzeService {
                 nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        ComponentLog componentLog = new ComponentLog();
-        if (kieWrapperComponentRules.isPresent()) {
-            List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
-                    allFileAttributes,
-                    notFoundManualRedactionEntries,
-                    document.streamAllImages().toList());
-
-            componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
-            log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        }
-
         RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
         EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
+                startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog,
-                entityLog, componentLog,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -316,26 +282,20 @@ public class AnalyzeService {
 
 
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
-                                           long startTime,
+                                           long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog,
-                                           EntityLog entityLog, ComponentLog componentLog,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
+        EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion);
-        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-        excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
 
-        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
+        if (entityLogChanges.isHasChanges()) {
-                analyzeRequest.getFileId(),
+            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges);
-                redactionLog,
+        }
-                analyzeRequest.getAnalysisNumber());
-        log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
+        log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         long duration = System.currentTimeMillis() - startTime;
 
@@ -346,13 +306,13 @@ public class AnalyzeService {
                 .fileId(analyzeRequest.getFileId())
                 .duration(duration)
                 .numberOfPages(numberOfPages)
-                .hasUpdates(redactionLogChange.isHasChanges())
+                .hasUpdates(entityLogChanges.isHasChanges())
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
                 .analysisNumber(analyzeRequest.getAnalysisNumber())
-                .rulesVersion(redactionLog.getRulesVersion())
+                .rulesVersion(entityLog.getRulesVersion())
-                .dictionaryVersion(redactionLog.getDictionaryVersion())
+                .dictionaryVersion(entityLog.getDictionaryVersion())
-                .legalBasisVersion(redactionLog.getLegalBasisVersion())
+                .legalBasisVersion(entityLog.getLegalBasisVersion())
-                .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
+                .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
                 .wasReanalyzed(isReanalysis)
                 .manualRedactions(analyzeRequest.getManualRedactions())
                 .addedFileAttributes(addedFileAttributes)
@@ -360,16 +320,58 @@ public class AnalyzeService {
     }
 
 
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                      KieWrapper kieWrapperComponentRules,
+                                                      Set<FileAttribute> addedFileAttributes,
+                                                      EntityLogChanges entityLogChanges) {
+
+        if (!kieWrapperComponentRules.isPresent()) {
+            return;
+        }
+
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                entityLogChanges.getEntityLog(),
+                addedFileAttributes.stream().toList());
+        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
+
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
+
+    }
+
+
+    private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
+
+        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
+        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
+        excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
+        EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                entityLog,
+                analyzeRequest.getAnalysisNumber());
+        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                redactionLog,
+                analyzeRequest.getAnalysisNumber());
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+        return entityLogChanges;
+    }
+
+
     private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
 
         return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList());
     }
 
 
-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, Document document, DictionaryIncrement dictionaryIncrement) {
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
 
         return analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, redactionLog, document, analyzeRequest) //
+                ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
                 : analyzeRequest.getSectionsToReanalyse();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -2,28 +2,30 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.List;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 
 @Service
 public class ComponentLogCreatorService {
 
-    public ComponentLog buildComponentLog(KieContainer container, int analysisNumber, List<Component> components) {
+    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
 
-        List<ComponentLogEntry> componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList();
+        List<ComponentLogCategory> componentLogCategories = components.stream()
-        return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1);
+                .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
+                .entrySet()
+                .stream()
+                .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue()))
+                .toList();
+        return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1);
     }
 
 
@@ -31,8 +33,7 @@ public class ComponentLogCreatorService {
 
         return ComponentLogEntry.builder()
                 .value(component.getValue())
-                .transformation(component.getTransformation())
+                .transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList())
-                .originalValue(component.getOriginalValue())
                 .componentEntityReferences(toComponentEntityReferences(component.getReferences()))
                 .build();
     }
@@ -40,45 +41,19 @@ public class ComponentLogCreatorService {
 
     private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
 
-        return references.stream().flatMap(this::toComponentEntityReference).toList();
+        return references.stream().map(this::toComponentEntityReference).toList();
     }
 
 
-    private Stream<ComponentEntityReference> toComponentEntityReference(Entity entity) {
+    private ComponentEntityReference toComponentEntityReference(Entity entity) {
 
-        if (entity instanceof TextEntity textEntity) {
+        return ComponentEntityReference.builder()
-            textEntity.getPositionsOnPagePerPage()
+                .id(entity.getId())
-                    .stream()
+                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0))
-                    .map(positionOnPage -> ComponentEntityReference.builder()
+                .reason(entity.getReason())
-                            .page(positionOnPage.getPage().getNumber())
+                .ruleIdentifier(entity.getMatchedRule())
-                            .type(textEntity.getType())
+                .type(entity.getType())
-                            .id(positionOnPage.getId())
+                .build();
-                            .ruleIdentifier(textEntity.getMatchedRule().getRuleIdentifier().toString())
-                            .reason(textEntity.buildReasonWithManualChangeDescriptions())
-                            .build());
-        } else if (entity instanceof Image image) {
-            return Stream.of(ComponentEntityReference.builder()
-                    .id(image.getId())
-                    .reason(image.buildReasonWithManualChangeDescriptions())
-                    .type(image.getImageType().toString())
-                    .ruleIdentifier(image.getMatchedRule().getRuleIdentifier().toString())
-                    .page(image.getFirstPage().getNumber())
-                    .build());
-        } else if (entity instanceof ManualEntity manualEntity) {
-            return manualEntity.getEntityPosition()
-                    .stream()
-                    .collect(Collectors.groupingBy(RectangleWithPage::pageNumber))
-                    .keySet()
-                    .stream()
-                    .map(pageNumber -> ComponentEntityReference.builder()
-                            .id(manualEntity.getId())
-                            .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                            .type(manualEntity.getType())
-                            .ruleIdentifier(manualEntity.getRuleIdentifier())
-                            .page(pageNumber)
-                            .build());
-        }
-        throw new UnsupportedOperationException(String.format("Entity with class %s is not implemented yet", entity.getClass()));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -0,0 +1,116 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class EntityChangeLogService {
+
+    private final RedactionStorageService redactionStorageService;
+
+
+    @Timed("redactmanager_computeChanges")
+    public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) {
+
+        long start = System.currentTimeMillis();
+
+        EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId);
+
+        if (previousEntityLog == null) {
+            currentEntityLog.getEntityLogEntry().forEach(entry -> {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            });
+            return new EntityLogChanges(currentEntityLog, false);
+        }
+
+        List<EntityLogEntry> previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
+
+        Map<String, EntityLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries);
+        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries);
+
+        List<EntityLogEntry> newEntityLogEntries = previousEntityLog.getEntityLogEntry();
+
+        List<EntityLogEntry> toRemove = new ArrayList<>();
+        newEntityLogEntries.forEach(entry -> {
+            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
+            } else if (removedIds.contains(entry.getId())) {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+            } else if (addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
+            }
+        });
+
+        newEntityLogEntries.removeAll(toRemove);
+
+        addedEntryIds.forEach((k, v) -> {
+            if (v.getChanges().isEmpty()) {
+                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            }
+            newEntityLogEntries.add(v);
+        });
+
+        currentEntityLog.setEntityLogEntry(newEntityLogEntries);
+
+        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
+        return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
+    }
+
+
+    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+
+        Set<EntityLogEntry> removed = new HashSet<>(previouslyExistingEntries);
+        currentEntityLog.getEntityLogEntry().forEach(removed::remove);
+
+        Set<String> removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        return removedIds;
+    }
+
+
+    private static Map<String, EntityLogEntry> getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+
+        Set<EntityLogEntry> currentExistingEntries = currentEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .collect(Collectors.toSet());
+
+        previouslyExistingEntries.forEach(currentExistingEntries::remove);
+        Map<String, EntityLogEntry> addedIds = new HashMap<>();
+        currentExistingEntries.forEach(entry -> {
+            addedIds.put(entry.getId(), entry);
+        });
+        return addedIds;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -188,13 +188,13 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryState buildEntryState(Entity entity) {
+    private EntryState buildEntryState(IEntity IEntity) {
 
-        if (entity.applied()) {
+        if (IEntity.applied()) {
             return EntryState.APPLIED;
-        } else if (entity.skipped()) {
+        } else if (IEntity.skipped()) {
             return EntryState.SKIPPED;
-        } else if (entity.ignored()) {
+        } else if (IEntity.ignored()) {
             return EntryState.IGNORED;
         } else {
             return EntryState.REMOVED;
@@ -202,13 +202,13 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryType buildEntryType(Entity entity, boolean isHint) {
+    private EntryType buildEntryType(IEntity IEntity, boolean isHint) {
 
-        if (entity instanceof TextEntity textEntity) {
+        if (IEntity instanceof TextEntity textEntity) {
             return getEntryType(isHint, textEntity.getEntityType());
-        } else if (entity instanceof ManualEntity manualEntity) {
+        } else if (IEntity instanceof ManualEntity manualEntity) {
             return getEntryType(isHint, manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
+        } else if (IEntity instanceof Image) {
             return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException("Entity subclass %s not implemented!");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -7,7 +7,7 @@ import java.util.NoSuchElementException;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@@ -24,14 +24,14 @@ public class ManualChangesApplicationService {
     private final EntityCreationService entityCreationService;
 
 
-    public void recategorize(Entity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
+    public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
 
-        if (entityToBeReCategorized instanceof Image image) {
+        if (IEntityToBeReCategorized instanceof Image image) {
             image.setImageType(ImageType.fromString(manualRecategorization.getType()));
             return;
         }
         // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
-        if (entityToBeReCategorized instanceof TextEntity textEntity) {
+        if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
             TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
             recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
             recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -1,5 +1,131 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.kie.api.runtime.KieSession;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
+import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ComponentCreationService {
 
+    KieSession kieSession;
+    Set<Entity> referencedEntities = new HashSet<>();
+
+
+    public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) {
+
+        String transformation = String.format("First found value or else '%s'", fallback);
+        String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
+        createComponent(ruleIdentifier, category, value, transformation, entities);
+    }
+
+
+    public void joining(String ruleIdentifier, String category, Collection<Entity> entities) {
+
+        joining(ruleIdentifier, category, entities, ", ");
+    }
+
+
+    public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
+
+        String transformation = String.format("Joining all values with '%s'", delimiter);
+        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
+        createComponent(ruleIdentifier, category, value, transformation, entities);
+    }
+
+
+    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
+
+        joining(ruleIdentifier, category, entities, ", ");
+    }
+
+
+    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
+
+        String transformation = String.format("Joining all values with '%s'", delimiter);
+        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
+        createComponent(ruleIdentifier, category, value, transformation, entities);
+    }
+
+
+    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) {
+
+        convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy");
+    }
+
+
+    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) {
+
+        String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
+        String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
+        createComponent(ruleIdentifier, category, date, transformation, entities);
+    }
+
+
+    public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
+
+        entities.forEach(entity -> createComponent(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
+    }
+
+
+    public void createComponent(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) {
+
+        referencedEntities.addAll(references);
+
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .category(category)
+                .value(value)
+                .transformation(transformation)
+                .references(new LinkedList<>(references)));
+    }
+
+
+    public void createComponent(String ruleIdentifier, String category, String value, String transformation, Entity reference) {
+
+        referencedEntities.add(reference);
+        List<Entity> referenceList = new LinkedList<>();
+        referenceList.add(reference);
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .category(category)
+                .value(value)
+                .transformation(transformation)
+                .references(referenceList)
+                .build());
+    }
+
+
+    public void createComponent(String ruleIdentifier, String category, String value, String transformation) {
+
+        createComponent(ruleIdentifier, category, value, transformation, Collections.emptyList());
+    }
+
+
+    public void createComponent(String ruleIdentifier, String category, String value) {
+
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .category(category)
+                .value(value)
+                .transformation("")
+                .references(Collections.emptyList())
+                .build());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java

@@ -0,0 +1,41 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+
+public abstract class EntityComparators implements Comparator<Entity> {
+
+    private static class LongestEntity implements Comparator<Entity> {
+
+        @Override
+        public int compare(Entity Entity, Entity otherEntity) {
+
+            return Integer.compare(Entity.getLength(), otherEntity.getLength());
+        }
+
+    }
+
+    private static class FirstEntity implements Comparator<Entity> {
+
+        @Override
+        public int compare(Entity Entity, Entity otherEntity) {
+
+            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
+        }
+
+    }
+
+
+    public static Comparator<Entity> length() {
+
+        return new LongestEntity();
+    }
+
+
+    public static Comparator<Entity> start() {
+
+        return new FirstEntity();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.service;
+package com.iqser.red.service.redaction.v1.server.service.document;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -8,9 +8,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import org.springframework.stereotype.Component;
+import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -19,13 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -34,18 +34,18 @@ import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-@Component
+@Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-class SectionFinderService {
+public class SectionFinderService {
 
     @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Document document, AnalyzeRequest analyzeRequest) {
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
 
         long start = System.currentTimeMillis();
         Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
         Set<Integer> sectionsToReanalyse = new HashSet<>();
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
+        for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
             if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
                 sectionsToReanalyse.add(entry.getSectionNumber());
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -1,16 +1,26 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
-import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
+import org.kie.api.runtime.KieContainer;
+import org.kie.api.runtime.KieSession;
+import org.kie.api.runtime.rule.QueryResults;
+import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -23,9 +33,61 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class ComponentDroolsExecutionService {
 
-    public List<Component> executeRules(Set<TextEntity> entities, List<FileAttribute> allFileAttributes, List<ManualEntity> notFoundManualRedactionEntries, List<Image> list) {
+    RedactionServiceSettings settings;
 
-        return Collections.emptyList();
+
+    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, List<FileAttribute> fileAttributes) {
+
+        KieSession kieSession = kieContainer.newKieSession();
+        ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
+
+        kieSession.setGlobal("componentCreationService", componentCreationService);
+        entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert);
+        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
+
+        CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
+            kieSession.fireAllRules();
+            return null;
+        });
+
+        try {
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+        } catch (ExecutionException e) {
+            kieSession.dispose();
+            if (e.getCause() instanceof TimeoutException) {
+                throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
+            }
+            throw new RuntimeException(e);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
+        List<Component> components = getComponents(kieSession);
+        kieSession.dispose();
+        return components;
+    }
+
+
+    public List<FileAttribute> getFileAttributes(KieSession kieSession) {
+
+        List<FileAttribute> fileAttributes = new LinkedList<>();
+        QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute"));
+        }
+        return fileAttributes;
+    }
+
+
+    public List<Component> getComponents(KieSession kieSession) {
+
+        List<Component> components = new LinkedList<>();
+        QueryResults entitiesResult = kieSession.getQueryResults("getComponents");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            components.add((Component) resultsRow.get("$component"));
+        }
+        return components;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -47,6 +47,16 @@ public class RedactionStorageService {
     }
 
 
+    @SneakyThrows
+    public File getStoredObjectFile(String storageId) {
+
+        File tempFile = File.createTempFile("temp", "data");
+        tempFile.deleteOnExit();
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
+    }
+
+
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@UtilityClass
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DateConverter {
+
+    static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH),
+            new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH),
+            new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH),
+            new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH),
+            new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH),
+            new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
+
+
+    public String convertDate(String dateAsString, String resultFormat) {
+
+        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
+        Date date = null;
+        for (SimpleDateFormat format : formats) {
+
+            try {
+                date = format.parse(dateAsString);
+                break;
+            } catch (Exception e) {
+                log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage());
+                // ignore, try next...
+            }
+        }
+
+        if (date == null) {
+            return dateAsString;
+        }
+
+        return resultDateFormat.format(date);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -43,6 +43,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/documine_flora.drl");
+    private static final String COMPONENT_RULES = loadFromClassPath("drools/documine_flora_components.drl");
 
 
     @Test
@@ -148,7 +149,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
 
         loadTypeForTest();
         loadNerForTest();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -13,7 +13,6 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -36,10 +35,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
-import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException;
 
 import lombok.RequiredArgsConstructor;
 
@@ -67,14 +66,14 @@ public class AnnotationService {
      */
     public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
 
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
+        var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
                 annotateRequest.getFileId(),
                 FileType.ORIGIN));
 
         var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
         var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
-        try (PDDocument pdDocument = Loader.loadPDF(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
+        try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
             dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java

@@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
-public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
+public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -255,7 +255,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
 
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf";
         var fileResource = new ClassPathResource(filename + ".pdf");
-        try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) {
+        try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
 
             for (Page page : document.getPages()) {
                 List<Rectangle2D> entityPositionsOnPage = page.getEntities()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java

@@ -79,7 +79,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_SEMANTIC_NODES_BBOX.pdf";
         var fileResource = new ClassPathResource(filename + ".pdf");
 
-        try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) {
+        try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
 
             PdfVisualisationUtility.drawDocumentGraph(pdDocument, document);
             PdfVisualisationUtility.drawTextBlock(pdDocument,

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.MockitoAnnotations;
 import org.springframework.boot.test.mock.mockito.MockBean;
@@ -154,4 +155,12 @@ class DroolsSyntaxValidationServiceTest {
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
+
+    @Test
+    @SneakyThrows
+    @Disabled
+    void fixAllRuleFiles() {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -71,7 +71,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
         assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end()));
 
         ClassPathResource resource = new ClassPathResource(filePath);
-        try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) {
+        try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
 
             Stream<NerEntities.NerEntity> unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
                     .getNerEntityList()
@@ -120,7 +120,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
         assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end()));
 
         ClassPathResource resource = new ClassPathResource(filePath);
-        try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) {
+        try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
 
             List<TextEntity> validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
                     .getNerEntityList()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -21,6 +21,7 @@ public class LayoutParsingRequestProvider {
         var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
         var pageFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES);
         var simplifiedTextStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT);
+        var viewerDocumentStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.VIEWER_DOCUMENT);
         return LayoutParsingRequest.builder()
                 .layoutParsingType(layoutParsingType)
                 .originFileStorageId(originFileStorageId)
@@ -31,7 +32,7 @@ public class LayoutParsingRequestProvider {
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
                 .sectionGridStorageId(sectionGridStorageId)
-                .simplifiedTextStorageId(simplifiedTextStorageId)
+                .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -0,0 +1,558 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Default Components rules ------------------------------------
+
+
+rule "StudyTitle.0.0: Study Title"
+    when
+        $titleCandidates: List() from collect (Entity(type == "study_title"))
+    then
+        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
+    end
+
+
+rule "PerformingLaboratory.0.0: Performing Laboratory"
+    when
+        $laboratoryName: Entity(type == "laboratory_name")
+        $laboratoryCountry: Entity(type == "laboratory_country", $distance: Math.abs($laboratoryName.startOffset - startOffset))
+        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < $distance)
+        eval($distance < 80)
+    then
+        componentCreationService.createComponent("PerformingLaboratory.0.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!");
+    end
+
+rule "PerformingLaboratory.0.1: Performing Laboratory"
+    when
+        $laboratoryName: Entity(type == "laboratory_name")
+        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
+    then
+        componentCreationService.createComponent("DefaultComponents.0.1", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!");
+    end
+
+rule "PerformingLaboratory.0.2: Performing Laboratory"
+    salience -1
+    when
+        not Component(category == "Performing_Laboratory")
+    then
+        componentCreationService.createComponent("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
+    end
+
+
+rule "ReportNumber.0.0: Report number"
+    when
+        $reportNumberCandidates: List() from collect (Entity(type == "report_number"))
+    then
+        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!");
+    end
+
+
+rule "GLPStudy.0.0: GLP Study"
+    when
+        $glpStudy: Entity(type() == "glp_study")
+    then
+        componentCreationService.createComponent("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy);
+    end
+
+rule "GLPStudy.1.0: GLP Study"
+    when
+        not Entity(type() == "glp_study")
+    then
+        componentCreationService.createComponent("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not");
+    end
+
+rule "GLPStudy.2.0: GLP Study"
+    when
+        $first: Component(category == "GLP_study")
+        $second: Component(category == "GLP_study", value == $first.value, this != $first)
+    then
+        $first.addReferences($second.getReferences());
+        retract($second);
+    end
+
+
+rule "DefaultComponents.4.0: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number")
+        not Entity(type == "oecd_guideline_year")
+    then
+        componentCreationService.createComponent("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!");
+    end
+
+rule "DefaultComponents.4.1: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2015")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.1",
+           "Test_Guidelines_1",
+           "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.2: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "403")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.2",
+           "Test_Guidelines_1",
+           "Nº 403: Acute Inhalation Toxicity (12/05/1981)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.3: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "442B")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.3",
+           "Test_Guidelines_1",
+           "Nº 442B: Skin Sensitization (27/06/2018)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.4: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "487")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.4",
+           "Test_Guidelines_1",
+           "Nº 487: Micronucleus Human Lymphocytes (2016)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.5: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2004")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "428")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.5",
+           "Test_Guidelines_1",
+           "Nº 428: Split-Thickness Skin test (2004)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.6: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "442A")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.6",
+           "Test_Guidelines_1",
+           "Nº 442A: Skin Sensitization (23/07/2018)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.7: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.7",
+           "Test_Guidelines_1",
+           "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.8: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "406")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.8",
+           "Test_Guidelines_1",
+           "Nº 406: Skin Sensitisation (1992)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.9: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "438")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.9",
+           "Test_Guidelines_1",
+           "Nº 438: Eye Irritation (26/06/2018)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.10: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "436")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.10",
+           "Test_Guidelines_1",
+           "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.11: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "429")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.11",
+           "Test_Guidelines_1",
+           "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.12: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2020")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "471")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.12",
+           "Test_Guidelines_1",
+           "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.13: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.13",
+           "Test_Guidelines_1",
+           "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.14: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1997")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "471")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.14",
+           "Test_Guidelines_1",
+           "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.15: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "403")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.15",
+           "Test_Guidelines_1",
+           "Nº 403: Acute Inhalation Toxicity (08/09/2009)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.16: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "433")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.16",
+           "Test_Guidelines_1",
+           "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.17: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2019")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "439")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.17",
+           "Test_Guidelines_1",
+           "Nº 439: Skin Irritation (2019)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.18: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2001")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "425")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.18",
+           "Test_Guidelines_1",
+           "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.19: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "402")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.19",
+           "Test_Guidelines_1",
+           "Nº 402: Acute Dermal Toxicity (09/10/2017)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.20: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.20",
+           "Test_Guidelines_1",
+           "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.21: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "474")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.21",
+           "Test_Guidelines_1",
+           "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.22: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "433")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.22",
+           "Test_Guidelines_1",
+           "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.23: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "404")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.23",
+           "Test_Guidelines_1",
+           "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.24: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2010")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "429")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.24",
+           "Test_Guidelines_1",
+           "Nº 429: Skin Sensitisation (23/07/2010)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.25: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.25",
+           "Test_Guidelines_1",
+           "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.26: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "402")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.26",
+           "Test_Guidelines_1",
+           "Nº 402: Acute Dermal Toxicity (24/02/1987)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.27: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2012")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.27",
+           "Test_Guidelines_1",
+           "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.28: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "405")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.28",
+           "Test_Guidelines_1",
+           "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+rule "DefaultComponents.4.29: Test Guideline 1"
+    when
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2008")
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == "425")
+    then
+        componentCreationService.createComponent(
+           "DefaultComponents.4.29",
+           "Test_Guidelines_1",
+           "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)",
+           "OECD Number and guideline year mapped!"
+        );
+    end
+
+
+rule "DefaultComponents.5.0: Test Guideline 2"
+    when
+        $epaGuideLines: List() from collect (Entity(type() == "epa_guideline"))
+        $ecGuideLines: List() from collect (Entity(type() == "ec_guideline"))
+    then
+        componentCreationService.joining("DefaultComponents.5.0",
+            "Test_Guideline_2",
+            Stream.of(
+              $epaGuideLines.stream(),
+              $ecGuideLines.stream())
+          .flatMap(a -> a)
+          .toList()
+        );
+    end
+
+
+rule "DefaultComponents.6.0: Experimental Starting Date"
+    when
+        $startDates: List(!isEmpty()) from collect (Entity(type() == "experimental_start_date"))
+    then
+        componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates);
+    end
+
+
+rule "DefaultComponents.7.0: Experimental Completion Date"
+    when
+        $endDates: List(!isEmpty()) from collect (Entity(type() == "experimental_end_date"))
+    then
+        componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates);
+    end
+
+
+rule "DefaultComponents.8.0: Certificate of analysis batch identification"
+    when
+        $batchNumbers: List(!isEmpty()) from collect (Entity(type() == "batch_number"))
+    then
+        componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers);
+    end
+
+
+rule "DefaultComponents.999.0: Create components for all unmapped entities."
+    salience -999
+    when
+        $allEntities: List(!isEmpty()) from collect (Entity()
+    then
+        componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
+    end
+
+
+
+//------------------------------------ Component merging rules ------------------------------------
+
+rule "X.0.0: merge duplicate component references"
+    when
+        $first: Component()
+        $duplicate: Component(this != $first, category == category, value == value)
+    then
+        $first.getReferences().addAll($duplicate.getReferences());
+        retract($duplicate);
+    end
+
+

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -58,6 +58,7 @@ global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
+
 //------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -58,6 +58,7 @@ global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
+
 // --------------------------------------- queries -------------------------------------------------------------------
 
 query "getFileAttributes"

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -47,7 +47,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType
 
 global Document document
 global EntityCreationService entityCreationService