Merge branch 'DM-285' into 'master'

DM-285: improve EntityLog change computation

Closes DM-285

See merge request redactmanager/redaction-service!134

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -18,8 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
@@ -33,10 +31,10 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@@ -118,12 +116,15 @@ public class AnalyzeService {
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
-        EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
+        EntityLog entityLog = entityLogCreatorService.createInitialEntityLogFromActiveEntities(analyzeRequest,
+                document,
+                notFoundManualRedactionEntries,
+                dictionary.getVersion(),
+                kieWrapperEntityRules.rulesVersion());
 
         return finalizeAnalysis(analyzeRequest,
                 startTime,
-                kieWrapperComponentRules,
+                kieWrapperComponentRules, new EntityLogChanges(entityLog, false),
-                entityLog,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionary.getVersion(),
@@ -132,42 +133,6 @@ public class AnalyzeService {
     }
 
 
-    private EntityLog createEntityLog(AnalyzeRequest analyzeRequest,
-                                      Document document,
-                                      List<ManualEntity> notFoundManualRedactionEntries,
-                                      Dictionary dictionary,
-                                      KieWrapper wrapper) {
-
-        List<EntityLogEntry> entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-
-        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                entityLogEntries,
-                toEntityLogLegalBasis(legalBasis),
-                dictionary.getVersion().getDossierTemplateVersion(),
-                dictionary.getVersion().getDossierVersion(),
-                wrapper.rulesVersion(),
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                entityLog.getEntityLogEntry(),
-                true);
-
-        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
-        return entityLog;
-    }
-
-
-    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
-
-        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
-    }
-
-
     @Timed("redactmanager_reanalyze")
     @SneakyThrows
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
@@ -197,7 +162,9 @@ public class AnalyzeService {
         if (sectionsToReAnalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest,
                     startTime,
-                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog, previousRedactionLog,
+                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                    new EntityLogChanges(previousEntityLog, false),
+                    previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
                     true,
@@ -227,10 +194,16 @@ public class AnalyzeService {
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
-        EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
+        EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
+                document,
+                notFoundManualRedactionEntries,
+                previousEntityLog,
+                sectionsToReanalyseIds);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog,
+                startTime,
+                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                entityLogChanges,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -264,39 +237,24 @@ public class AnalyzeService {
     }
 
 
-    private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
-                                              Document document,
-                                              List<ManualEntity> notFoundManualRedactionEntries,
-                                              EntityLog previousEntityLog,
-                                              Set<Integer> sectionsToReanalyseIds) {
 
-        List<EntityLogEntry> newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newEntityLogEntries,
-                false);
-
-        previousEntityLog.getEntityLogEntry()
-                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
-        previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
-        return previousEntityLog;
-    }
 
 
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
-                                           KieWrapper kieWrapperComponentRules,
+                                           KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges,
-                                           EntityLog entityLog,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion);
+        finalizeRedactionLog(analyzeRequest, redactionLog, dictionaryVersion);
 
+        EntityLog entityLog = entityLogChanges.getEntityLog();
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         if (entityLogChanges.isHasChanges() || !isReanalysis) {
             computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges, dictionaryVersion);
         }
@@ -317,7 +275,6 @@ public class AnalyzeService {
                 .analysisNumber(analyzeRequest.getAnalysisNumber())
                 .rulesVersion(entityLog.getRulesVersion())
                 .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
-                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
                 .dictionaryVersion(entityLog.getDictionaryVersion())
                 .legalBasisVersion(entityLog.getLegalBasisVersion())
                 .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
@@ -353,32 +310,20 @@ public class AnalyzeService {
     }
 
 
-    private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest,
+    private RedactionLogChanges finalizeRedactionLog(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
-                                               EntityLog entityLog,
+        // TODO: remove redactionLog related stuff
-                                               RedactionLog redactionLog,
-                                               DictionaryVersion dictionaryVersion) {
 
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
 
-        // TODO: remove redactionLog related stuff
         redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
 
         excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
         RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber());
 
-        entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
-        entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-
-        excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages());
-
-        EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(previousEntityLog, entityLog, analyzeRequest.getAnalysisNumber());
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+
-        return entityLogChanges;
+        return redactionLogChange;
     }
 
 
@@ -492,16 +437,4 @@ public class AnalyzeService {
     }
 
 
-    private void excludeExcludedPages(EntityLog entityLog, Set<Integer> excludedPages) {
-
-        if (excludedPages != null && !excludedPages.isEmpty()) {
-            entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> {
-                if (excludedPages.contains(pos.getPageNumber())) {
-                    entry.setExcluded(true);
-                }
-            }));
-        }
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,11 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -13,10 +9,8 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
@@ -27,91 +21,59 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class EntityChangeLogService {
 
-    private final RedactionStorageService redactionStorageService;
-
 
     @Timed("redactmanager_computeChanges")
-    public EntityLogChanges computeChanges(EntityLog previousEntityLog, EntityLog currentEntityLog, int analysisNumber) {
+    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> currentEntityLogEntries, int analysisNumber) {
 
-        long start = System.currentTimeMillis();
+        boolean hasChanges = false;
 
-        if (previousEntityLog == null) {
+        var now = OffsetDateTime.now();
-            currentEntityLog.getEntityLogEntry().forEach(entry -> {
+        for (EntityLogEntry entityLogEntry : currentEntityLogEntries) {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
-            });
+            if (optionalPreviousEntity.isEmpty()) {
-            return new EntityLogChanges(currentEntityLog, false);
+                hasChanges = true;
+                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
+                continue;
+            }
+            var previousEntity = optionalPreviousEntity.get();
+            if (!previousEntity.getState().equals(entityLogEntry.getState())) {
+                hasChanges = true;
+                ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
+                if (changeType != null) {
+                    entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
+                }
+            }
         }
-
+        Set<String> existingIds = currentEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
-        List<EntityLogEntry> previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).toList();
+        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
-
+        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
-        Map<String, EntityLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries);
+        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
-        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries);
+        return hasChanges;
-
-        List<EntityLogEntry> newEntityLogEntries = previousEntityLog.getEntityLogEntry();
-
-        List<EntityLogEntry> toRemove = new ArrayList<>();
-        newEntityLogEntries.forEach(entry -> {
-            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            } else if (removedIds.contains(entry.getId())) {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-            } else if (addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            }
-        });
-
-        newEntityLogEntries.removeAll(toRemove);
-
-        addedEntryIds.forEach((k, v) -> {
-            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            }
-            newEntityLogEntries.add(v);
-        });
-
-        currentEntityLog.setEntityLogEntry(newEntityLogEntries);
-
-        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
-        return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
     }
 
 
-    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+    private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
-        Set<EntityLogEntry> removed = new HashSet<>(previouslyExistingEntries);
+        if (!isRemoved(previousState) && isRemoved(state)) {
-        currentEntityLog.getEntityLogEntry().forEach(removed::remove);
+            return ChangeType.REMOVED;
-
+        }
-        Set<String> removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        if (isRemoved(previousState) && !isRemoved(state)) {
-        return removedIds;
+            return ChangeType.ADDED;
+        }
+        if (previousState.equals(EntryState.APPLIED) && state.equals(EntryState.SKIPPED)) {
+            return ChangeType.CHANGED;
+        }
+        if (previousState.equals(EntryState.SKIPPED) && state.equals(EntryState.APPLIED)) {
+            return ChangeType.CHANGED;
+        }
+        return null;
     }
 
 
-    private static Map<String, EntityLogEntry> getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+    private static boolean isRemoved(EntryState state) {
 
-        Set<EntityLogEntry> currentExistingEntries = currentEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).collect(Collectors.toSet());
+        return state.equals(EntryState.IGNORED) || state.equals(EntryState.REMOVED);
-
-        previouslyExistingEntries.forEach(currentExistingEntries::remove);
-        Map<String, EntityLogEntry> addedIds = new HashMap<>();
-        currentExistingEntries.forEach(entry -> {
-            addedIds.put(entry.getId(), entry);
-        });
-        return addedIds;
     }
 
 
-    private static boolean lastChangeIsRemoved(EntityLogEntry entry) {
-
-        return entry.getChanges().stream().reduce((a, b) -> b).map(change -> change.getType().equals(ChangeType.REMOVED)).orElse(false);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,19 +1,31 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
@@ -32,19 +44,97 @@ public class EntityLogCreatorService {
 
     private final DictionaryService dictionaryService;
     private final ManualChangeFactory manualChangeFactory;
+    private final ImportedRedactionService importedRedactionService;
+    private final RedactionServiceSettings redactionServiceSettings;
+    private final LegalBasisClient legalBasisClient;
+    private final EntityChangeLogService entityChangeLogService;
 
 
-    public List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
+    public EntityLog createInitialEntityLogFromActiveEntities(AnalyzeRequest analyzeRequest,
+                                                              Document document,
+                                                              List<ManualEntity> notFoundManualRedactionEntries,
+                                                              DictionaryVersion dictionaryVersion,
+                                                              long rulesVersion) {
+
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntriesFromActiveEntities(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+
+        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                analyzeRequest.getAnalysisNumber(),
+                entityLogEntries,
+                toEntityLogLegalBasis(legalBasis),
+                dictionaryVersion.getDossierTemplateVersion(),
+                dictionaryVersion.getDossierVersion(),
+                rulesVersion,
+                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                entityLog.getEntityLogEntry(),
+                true);
+
+        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
+        var now = OffsetDateTime.now();
+        entityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analyzeRequest.getAnalysisNumber(), ChangeType.ADDED, now)));
+        excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages());
+        return entityLog;
+    }
+
+
+    public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
+                                                    Document document,
+                                                    List<ManualEntity> notFoundManualRedactionEntries,
+                                                    EntityLog previousEntityLog,
+                                                    Set<Integer> sectionsToReanalyseIds) {
+
+        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+        List<EntityLogEntry> previousEntries = previousEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
+                .toList();
+        previousEntityLog.getEntityLogEntry().removeAll(previousEntries);
+        boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                newEntityLogEntries,
+                false);
+
+        previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
+        previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
+        excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
+
+        return new EntityLogChanges(previousEntityLog, hasChanges);
+    }
+
+
+    private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<EntityLogEntry> entries = new ArrayList<>();
         document.getEntities()
                 .stream()
                 .filter(EntityLogCreatorService::isEntityOrRecommendationType)
-                .filter(entity -> !entity.removed())
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        document.streamAllImages().forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
         notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
+
+    }
+
+
+    private List<EntityLogEntry> createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
+
+        List<EntityLogEntry> entries = new ArrayList<>();
+        document.getEntities()
+                .stream()
+                .filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active)
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
+        document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
+        return entries;
     }
 
 
@@ -70,8 +160,7 @@ public class EntityLogCreatorService {
             EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId);
             redactionLogEntry.setId(positionOnPage.getId());
 
-            List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
+            List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
-                    .stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
             redactionLogEntry.setPositions(rectanglesPerLine);
@@ -95,12 +184,10 @@ public class EntityLogCreatorService {
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                 .type(entity.getType())
                 .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
-                .sectionNumber(sectionNumber)
+                .sectionNumber(sectionNumber).matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry())
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry())
                 .textAfter(entity.getTextAfter())
                 .textBefore(entity.getTextBefore())
-                .startOffset(entity.getTextRange().start())
+                .startOffset(entity.getTextRange().start()).endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry())
                 .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
                 .reference(referenceIds)
                 .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
@@ -110,15 +197,14 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         boolean isHint = isHint(type, dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(manualEntity.getId())
                 .color(getColor(type, dossierTemplateId, manualEntity.applied()))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions()).legalBasis(manualEntity.legalBasis()).value(manualEntity.value())
-                .legalBasis(manualEntity.legalBasis()).value(manualEntity.value())
                 .type(type)
                 .state(buildEntryState(manualEntity))
                 .entryType(buildEntryType(manualEntity, isHint))
@@ -144,14 +230,16 @@ public class EntityLogCreatorService {
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
         return EntityLogEntry.builder()
-                .id(image.getId()).value(image.value())
+                .id(image.getId())
+                .value(image.value())
                 .color(getColor(imageType, dossierTemplateId, image.applied()))
                 .value(image.value())
                 .type(imageType)
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false).positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .dictionaryEntry(false)
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
                 .sectionNumber(image.getTreeId().get(0))
                 .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
@@ -180,9 +268,9 @@ public class EntityLogCreatorService {
 
     private EntryState buildEntryState(IEntity entity) {
 
-        if (entity.applied()) {
+        if (entity.applied() && entity.active()) {
             return EntryState.APPLIED;
-        } else if (entity.skipped()) {
+        } else if (entity.skipped() && entity.active()) {
             return EntryState.SKIPPED;
         } else if (entity.ignored()) {
             return EntryState.IGNORED;
@@ -218,4 +306,23 @@ public class EntityLogCreatorService {
         };
     }
 
+
+    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
+
+        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+    }
+
+
+    private void excludeExcludedPages(EntityLog entityLog, Set<Integer> excludedPages) {
+
+        if (excludedPages != null && !excludedPages.isEmpty()) {
+            entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> {
+                if (excludedPages.contains(pos.getPageNumber())) {
+                    entry.setExcluded(true);
+                }
+            }));
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -45,7 +45,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
 
     @Test
-    @Disabled
+//    @Disabled
     public void titleExtraction() throws IOException {
 
         AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/ProblemDocs/8.SYN524464 FS (A16148F) - Teste de Ames (1).pdf");
@@ -59,7 +59,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
@@ -112,7 +112,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -1,8 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
-import static org.wildfly.common.Assert.assertFalse;
-import static org.wildfly.common.Assert.assertTrue;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -28,15 +27,15 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@@ -111,13 +110,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry1 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow();
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow();
 
-        // works in intellij, but not mvn install, but it works in UI so idk...
+        assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
-        // assertFalse(asyaLyon1.isRedacted());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
 
@@ -125,12 +123,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry2 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow();
 
-        assertTrue(asyaLyon2.isRedacted());
+        assertEquals(EntryState.APPLIED, asyaLyon2.getState());
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -143,9 +141,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    private Stream<RedactionLogEntry> findEntityByTypeAndValueAndSectionNumber(RedactionLog redactionLog, String type, String value, int sectionNumber) {
+    private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, int sectionNumber) {
 
-        return redactionLog.getRedactionLogEntry()
+        return redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
                 .filter(entry -> entry.getValue().equals(value))
@@ -153,9 +151,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    private static Stream<RedactionLogEntry> findEntityByTypeAndValue(RedactionLog redactionLog, String type, String value) {
+    private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
 
-        return redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
+        return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
     }
 
 
@@ -168,9 +166,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation())
+        var desireeEtAl = findEntityByTypeAndValue(entityLog, "CBI_author", "Desiree").filter(e -> e.getEntryType().equals(EntryType.ENTITY))
                 .filter(e -> e.getMatchedRule().startsWith("CBI.16"))
                 .findAny()
                 .orElseThrow();
@@ -180,9 +178,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         analyzeService.reanalyze(request);
         System.out.println("Finished reanalysis");
 
-        var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> !entry.isRecommendation())
+        assertEquals(EntryState.IGNORED,
-                .findFirst().get().getChanges().get(1).getType().equals(ChangeType.REMOVED));
+                findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState());
     }