diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 7c2063c4..4bff8a03 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -18,8 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; @@ -33,10 +31,10 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; @@ -118,12 +116,15 @@ public class AnalyzeService { log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); - EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); + EntityLog entityLog = entityLogCreatorService.createInitialEntityLogFromActiveEntities(analyzeRequest, + document, + notFoundManualRedactionEntries, + dictionary.getVersion(), + kieWrapperEntityRules.rulesVersion()); return finalizeAnalysis(analyzeRequest, startTime, - kieWrapperComponentRules, - entityLog, + kieWrapperComponentRules, new EntityLogChanges(entityLog, false), redactionLog, document.getNumberOfPages(), dictionary.getVersion(), @@ -132,42 +133,6 @@ public class AnalyzeService { } - private EntityLog createEntityLog(AnalyzeRequest analyzeRequest, - Document document, - List notFoundManualRedactionEntries, - Dictionary dictionary, - KieWrapper wrapper) { - - List entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); - - List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - - EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - entityLogEntries, - toEntityLogLegalBasis(legalBasis), - dictionary.getVersion().getDossierTemplateVersion(), - dictionary.getVersion().getDossierVersion(), - wrapper.rulesVersion(), - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - - List importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - entityLog.getEntityLogEntry(), - true); - - entityLog.setEntityLogEntry(importedRedactionFilteredEntries); - return entityLog; - } - - - private List toEntityLogLegalBasis(List legalBasis) { - - return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); - } - - @Timed("redactmanager_reanalyze") @SneakyThrows public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { @@ -197,7 +162,9 @@ public class AnalyzeService { if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, - kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog, previousRedactionLog, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), + new EntityLogChanges(previousEntityLog, false), + previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, @@ -227,10 +194,16 @@ public class AnalyzeService { log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); - EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds); + EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, + document, + notFoundManualRedactionEntries, + previousEntityLog, + sectionsToReanalyseIds); return finalizeAnalysis(analyzeRequest, - startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog, + startTime, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), + entityLogChanges, redactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -264,39 +237,24 @@ public class AnalyzeService { } - private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest, - Document document, - List notFoundManualRedactionEntries, - EntityLog previousEntityLog, - Set sectionsToReanalyseIds) { - List newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); - - var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - newEntityLogEntries, - false); - - previousEntityLog.getEntityLogEntry() - .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); - previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); - return previousEntityLog; - } private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, - KieWrapper kieWrapperComponentRules, - EntityLog entityLog, + KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, boolean isReanalysis, Set addedFileAttributes) { - EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion); + finalizeRedactionLog(analyzeRequest, redactionLog, dictionaryVersion); + EntityLog entityLog = entityLogChanges.getEntityLog(); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog()); + + log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (entityLogChanges.isHasChanges() || !isReanalysis) { computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges, dictionaryVersion); } @@ -317,7 +275,6 @@ public class AnalyzeService { .analysisNumber(analyzeRequest.getAnalysisNumber()) .rulesVersion(entityLog.getRulesVersion()) .componentRulesVersion(kieWrapperComponentRules.rulesVersion()) - .componentRulesVersion(kieWrapperComponentRules.rulesVersion()) .dictionaryVersion(entityLog.getDictionaryVersion()) .legalBasisVersion(entityLog.getLegalBasisVersion()) .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion()) @@ -353,32 +310,20 @@ public class AnalyzeService { } - private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, - EntityLog entityLog, - RedactionLog redactionLog, - DictionaryVersion dictionaryVersion) { + private RedactionLogChanges finalizeRedactionLog(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) { + // TODO: remove redactionLog related stuff - EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - // TODO: remove redactionLog related stuff redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber()); - entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); - entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); - - excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages()); - - EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(previousEntityLog, entityLog, analyzeRequest.getAnalysisNumber()); - log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog()); - return entityLogChanges; + + return redactionLogChange; } @@ -492,16 +437,4 @@ public class AnalyzeService { } - private void excludeExcludedPages(EntityLog entityLog, Set excludedPages) { - - if (excludedPages != null && !excludedPages.isEmpty()) { - entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { - if (excludedPages.contains(pos.getPageNumber())) { - entry.setExcluded(true); - } - })); - } - - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index 283a606a..028a2273 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -1,11 +1,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -13,10 +9,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; @@ -27,91 +21,59 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class EntityChangeLogService { - private final RedactionStorageService redactionStorageService; - @Timed("redactmanager_computeChanges") - public EntityLogChanges computeChanges(EntityLog previousEntityLog, EntityLog currentEntityLog, int analysisNumber) { + public boolean computeChanges(List previousEntityLogEntries, List currentEntityLogEntries, int analysisNumber) { - long start = System.currentTimeMillis(); + boolean hasChanges = false; - if (previousEntityLog == null) { - currentEntityLog.getEntityLogEntry().forEach(entry -> { - entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); - }); - return new EntityLogChanges(currentEntityLog, false); + var now = OffsetDateTime.now(); + for (EntityLogEntry entityLogEntry : currentEntityLogEntries) { + var optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny(); + if (optionalPreviousEntity.isEmpty()) { + hasChanges = true; + entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); + continue; + } + var previousEntity = optionalPreviousEntity.get(); + if (!previousEntity.getState().equals(entityLogEntry.getState())) { + hasChanges = true; + ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); + if (changeType != null) { + entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); + } + } } - - List previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).toList(); - - Map addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries); - Set removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries); - - List newEntityLogEntries = previousEntityLog.getEntityLogEntry(); - - List toRemove = new ArrayList<>(); - newEntityLogEntries.forEach(entry -> { - if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) { - List changes = entry.getChanges(); - changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now())); - var newEntry = addedEntryIds.get(entry.getId()); - newEntry.setChanges(changes); - addedEntryIds.put(entry.getId(), newEntry); - toRemove.add(entry); - } else if (removedIds.contains(entry.getId())) { - entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); - } else if (addedEntryIds.containsKey(entry.getId())) { - List changes = entry.getChanges(); - changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); - var newEntry = addedEntryIds.get(entry.getId()); - newEntry.setChanges(changes); - addedEntryIds.put(entry.getId(), newEntry); - toRemove.add(entry); - } - }); - - newEntityLogEntries.removeAll(toRemove); - - addedEntryIds.forEach((k, v) -> { - if (v.getChanges().isEmpty()) { - v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); - } - newEntityLogEntries.add(v); - }); - - currentEntityLog.setEntityLogEntry(newEntityLogEntries); - - log.debug("Change computation took: {}", System.currentTimeMillis() - start); - return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty()); + Set existingIds = currentEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + List removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList(); + removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); + removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); + return hasChanges; } - private static Set getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + private ChangeType calculateChangeType(EntryState state, EntryState previousState) { - Set removed = new HashSet<>(previouslyExistingEntries); - currentEntityLog.getEntityLogEntry().forEach(removed::remove); - - Set removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); - return removedIds; + if (!isRemoved(previousState) && isRemoved(state)) { + return ChangeType.REMOVED; + } + if (isRemoved(previousState) && !isRemoved(state)) { + return ChangeType.ADDED; + } + if (previousState.equals(EntryState.APPLIED) && state.equals(EntryState.SKIPPED)) { + return ChangeType.CHANGED; + } + if (previousState.equals(EntryState.SKIPPED) && state.equals(EntryState.APPLIED)) { + return ChangeType.CHANGED; + } + return null; } - private static Map getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + private static boolean isRemoved(EntryState state) { - Set currentExistingEntries = currentEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).collect(Collectors.toSet()); - - previouslyExistingEntries.forEach(currentExistingEntries::remove); - Map addedIds = new HashMap<>(); - currentExistingEntries.forEach(entry -> { - addedIds.put(entry.getId(), entry); - }); - return addedIds; + return state.equals(EntryState.IGNORED) || state.equals(EntryState.REMOVED); } - private static boolean lastChangeIsRemoved(EntityLogEntry entry) { - - return entry.getChanges().stream().reduce((a, b) -> b).map(change -> change.getType().equals(ChangeType.REMOVED)).orElse(false); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 61c04585..9cd54d27 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -1,19 +1,31 @@ package com.iqser.red.service.redaction.v1.server.service; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; @@ -32,19 +44,97 @@ public class EntityLogCreatorService { private final DictionaryService dictionaryService; private final ManualChangeFactory manualChangeFactory; + private final ImportedRedactionService importedRedactionService; + private final RedactionServiceSettings redactionServiceSettings; + private final LegalBasisClient legalBasisClient; + private final EntityChangeLogService entityChangeLogService; - public List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { + public EntityLog createInitialEntityLogFromActiveEntities(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + DictionaryVersion dictionaryVersion, + long rulesVersion) { + + List entityLogEntries = createEntityLogEntriesFromActiveEntities(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + + List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); + + EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionaryVersion.getDossierTemplateVersion(), + dictionaryVersion.getDossierVersion(), + rulesVersion, + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + + List importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + entityLog.getEntityLogEntry(), + true); + + entityLog.setEntityLogEntry(importedRedactionFilteredEntries); + var now = OffsetDateTime.now(); + entityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analyzeRequest.getAnalysisNumber(), ChangeType.ADDED, now))); + excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages()); + return entityLog; + } + + + public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + EntityLog previousEntityLog, + Set sectionsToReanalyseIds) { + + List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + List previousEntries = previousEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) + .toList(); + previousEntityLog.getEntityLogEntry().removeAll(previousEntries); + boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + + var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + newEntityLogEntries, + false); + + previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); + previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); + excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages()); + + return new EntityLogChanges(previousEntityLog, hasChanges); + } + + + private List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); document.getEntities() .stream() .filter(EntityLogCreatorService::isEntityOrRecommendationType) - .filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); - document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + document.streamAllImages().forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); return entries; + + } + + + private List createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { + + List entries = new ArrayList<>(); + document.getEntities() + .stream() + .filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); + document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); + return entries; } @@ -70,8 +160,7 @@ public class EntityLogCreatorService { EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId); redactionLogEntry.setId(positionOnPage.getId()); - List rectanglesPerLine = positionOnPage.getRectanglePerLine() - .stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) + List rectanglesPerLine = positionOnPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) .toList(); redactionLogEntry.setPositions(rectanglesPerLine); @@ -95,12 +184,10 @@ public class EntityLogCreatorService { .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.getType()) .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) - .sectionNumber(sectionNumber) - .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry()) + .sectionNumber(sectionNumber).matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry()) .textAfter(entity.getTextAfter()) .textBefore(entity.getTextBefore()) - .startOffset(entity.getTextRange().start()) - .endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .startOffset(entity.getTextRange().start()).endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) @@ -110,15 +197,14 @@ public class EntityLogCreatorService { } - public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); boolean isHint = isHint(type, dossierTemplateId); return EntityLogEntry.builder() .id(manualEntity.getId()) .color(getColor(type, dossierTemplateId, manualEntity.applied())) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()).value(manualEntity.value()) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()).legalBasis(manualEntity.legalBasis()).value(manualEntity.value()) .type(type) .state(buildEntryState(manualEntity)) .entryType(buildEntryType(manualEntity, isHint)) @@ -144,14 +230,16 @@ public class EntityLogCreatorService { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); return EntityLogEntry.builder() - .id(image.getId()).value(image.value()) + .id(image.getId()) + .value(image.value()) .color(getColor(imageType, dossierTemplateId, image.applied())) .value(image.value()) .type(imageType) .reason(image.buildReasonWithManualChangeDescriptions()) .legalBasis(image.legalBasis()) .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(false).positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) + .dictionaryEntry(false) + .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) .sectionNumber(image.getTreeId().get(0)) .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) @@ -180,9 +268,9 @@ public class EntityLogCreatorService { private EntryState buildEntryState(IEntity entity) { - if (entity.applied()) { + if (entity.applied() && entity.active()) { return EntryState.APPLIED; - } else if (entity.skipped()) { + } else if (entity.skipped() && entity.active()) { return EntryState.SKIPPED; } else if (entity.ignored()) { return EntryState.IGNORED; @@ -218,4 +306,23 @@ public class EntityLogCreatorService { }; } + + private List toEntityLogLegalBasis(List legalBasis) { + + return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); + } + + + private void excludeExcludedPages(EntityLog entityLog, Set excludedPages) { + + if (excludedPages != null && !excludedPages.isEmpty()) { + entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPageNumber())) { + entry.setExcluded(true); + } + })); + } + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 1df01fda..e56944fd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -45,7 +45,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test - @Disabled +// @Disabled public void titleExtraction() throws IOException { AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/ProblemDocs/8.SYN524464 FS (A16148F) - Teste de Ames (1).pdf"); @@ -59,7 +59,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - + var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf"; @@ -112,7 +112,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 8dc59459..dfedeb21 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -1,8 +1,7 @@ package com.iqser.red.service.redaction.v1.server; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; -import static org.wildfly.common.Assert.assertFalse; -import static org.wildfly.common.Assert.assertTrue; import java.io.FileOutputStream; import java.io.IOException; @@ -28,15 +27,15 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -111,13 +110,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry1 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow(); + var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow(); - // works in intellij, but not mvn install, but it works in UI so idk... - // assertFalse(asyaLyon1.isRedacted()); + assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); @@ -125,12 +123,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { request.setManualRedactions(manualRedactions); analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry2 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow(); + var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow(); - assertTrue(asyaLyon2.isRedacted()); + assertEquals(EntryState.APPLIED, asyaLyon2.getState()); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -143,9 +141,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } - private Stream findEntityByTypeAndValueAndSectionNumber(RedactionLog redactionLog, String type, String value, int sectionNumber) { + private Stream findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, int sectionNumber) { - return redactionLog.getRedactionLogEntry() + return redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals(type)) .filter(entry -> entry.getValue().equals(value)) @@ -153,9 +151,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } - private static Stream findEntityByTypeAndValue(RedactionLog redactionLog, String type, String value) { + private static Stream findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) { - return redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); + return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); } @@ -168,9 +166,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation()) + var desireeEtAl = findEntityByTypeAndValue(entityLog, "CBI_author", "Desiree").filter(e -> e.getEntryType().equals(EntryType.ENTITY)) .filter(e -> e.getMatchedRule().startsWith("CBI.16")) .findAny() .orElseThrow(); @@ -180,9 +178,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); System.out.println("Finished reanalysis"); - var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> !entry.isRecommendation()) - .findFirst().get().getChanges().get(1).getType().equals(ChangeType.REMOVED)); + var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertEquals(EntryState.IGNORED, + findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState()); }