diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index aec0d4e0..8840888b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; @@ -18,7 +19,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; @@ -26,6 +26,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; @@ -67,13 +68,13 @@ public class EntityLogCreatorService { List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - entityLogEntries, - toEntityLogLegalBasis(legalBasis), - dictionaryVersion.getDossierTemplateVersion(), - dictionaryVersion.getDossierVersion(), - rulesVersion, - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionaryVersion.getDossierTemplateVersion(), + dictionaryVersion.getDossierVersion(), + rulesVersion, + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); @@ -114,21 +115,24 @@ public class EntityLogCreatorService { DictionaryVersion dictionaryVersion) { List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream() - .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) + .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() + .get(0))) .collect(Collectors.toList()); - Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + Set newEntityIds = newEntityLogEntries.stream() + .map(EntityLogEntry::getId) + .collect(Collectors.toSet()); List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() .stream() .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() - .get(0)))) + .get(0)))) .toList(); previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, - newEntityLogEntries, - analyzeRequest.getManualRedactions(), - analyzeRequest.getAnalysisNumber()); + newEntityLogEntries, + analyzeRequest.getManualRedactions(), + analyzeRequest.getAnalysisNumber()); previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); @@ -147,8 +151,12 @@ public class EntityLogCreatorService { .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); - document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); - notFoundPrecursorEntries.stream().filter(entity -> !entity.removed()).forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId))); + document.streamAllImages() + .filter(entity -> !entity.removed()) + .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + notFoundPrecursorEntries.stream() + .filter(entity -> !entity.removed()) + .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId))); return entries; } @@ -192,11 +200,13 @@ public class EntityLogCreatorService { .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) .containingNodeId(image.getTreeId()) .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) - .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) + .section(image.getManualOverwrite().getSection() + .orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) .state(buildEntryState(image)) .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE) + .engines(getEngines(null, image.getManualOverwrite())) .build(); } @@ -204,7 +214,8 @@ public class EntityLogCreatorService { private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) { - String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType()); + String type = precursorEntity.getManualOverwrite().getType() + .orElse(precursorEntity.getType()); boolean isHint = isHint(precursorEntity.getEntityType()); return EntityLogEntry.builder() .id(precursorEntity.getId()) @@ -214,7 +225,8 @@ public class EntityLogCreatorService { .type(type) .state(buildEntryState(precursorEntity)) .entryType(buildEntryType(precursorEntity)) - .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection())) + .section(precursorEntity.getManualOverwrite().getSection() + .orElse(precursorEntity.getSection())) .containingNodeId(Collections.emptyList()) .closestHeadline("") .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) @@ -224,13 +236,12 @@ public class EntityLogCreatorService { .textBefore("") .startOffset(-1) .endOffset(-1) - .positions(precursorEntity.getManualOverwrite() - .getPositions() - .orElse(precursorEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) - .engines(precursorEntity.getEngines()) + .positions(precursorEntity.getManualOverwrite().getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) + .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite())) //imported is no longer used, frontend should check engines //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) .imported(false) @@ -243,14 +254,20 @@ public class EntityLogCreatorService { private EntityLogEntry createEntityLogEntry(TextEntity entity) { Set referenceIds = new HashSet<>(); - entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); + entity.references() + .stream() + .filter(TextEntity::active) + .forEach(ref -> ref.getPositionsOnPagePerPage() + .forEach(pos -> referenceIds.add(pos.getId()))); boolean isHint = isHint(entity.getEntityType()); return EntityLogEntry.builder() .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) - .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .value(entity.getManualOverwrite().getValue() + .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.type()) - .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) + .section(entity.getManualOverwrite().getSection() + .orElse(entity.getDeepestFullyContainingNode().toString())) .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) @@ -260,7 +277,7 @@ public class EntityLogCreatorService { .startOffset(entity.getTextRange().start()) .endOffset(entity.getTextRange().end()) .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + .engines(getEngines(entity.getEngines(), entity.getManualOverwrite())) //imported is no longer used, frontend should check engines //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) .imported(false) @@ -272,6 +289,17 @@ public class EntityLogCreatorService { } + private Set getEngines(Set currentEngines, ManualChangeOverwrite manualChangeOverwrite) { + + Set engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>(); + + if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) { + engines.add(Engine.MANUAL); + } + return engines; + } + + private boolean isHint(EntityType entityType) { return entityType.equals(EntityType.HINT); @@ -317,7 +345,9 @@ public class EntityLogCreatorService { private List toEntityLogLegalBasis(List legalBasis) { - return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); + return legalBasis.stream() + .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())) + .collect(Collectors.toList()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 128849f7..c483a49a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; @@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; @@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .filter(entry -> entry.getValue().equals("Oxford University Press")) .findFirst() .get(); + assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL)); var asyaLyon = redactionLog.getEntityLogEntry() .stream() @@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { assertEquals(asyaLyon2.getState(), EntryState.APPLIED); assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size()); + assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL)); } @@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); - EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow(); + EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getId().equals(annotationId)) + .findFirst() + .orElseThrow(); assertEquals("Expand to Hint", entityLogEntry.getValue()); assertEquals(1, entityLogEntry.getPositions().size()); - assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertEquals(ManualRedactionType.RESIZE, + entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL)); + } + + + @Test + @SneakyThrows + public void testAddEngineManualToResizeDictionaryEntry() { + + String filePath = "files/new/crafted document.pdf"; + AnalyzeRequest request = uploadFileToStorage(filePath); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + AnalyzeResult result = analyzeService.analyze(request); + ManualRedactions manualRedactions = new ManualRedactions(); + + EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); + var dictionaryEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()) + .findFirst() + .get(); + ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() + .annotationId(dictionaryEntry.getId()) + .requestDate(OffsetDateTime.now()) + .value("Image") + .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1))) + .updateDictionary(true) + .build(); + manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction)); + request.setManualRedactions(manualRedactions); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); + EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getId().equals(dictionaryEntry.getId())) + .findFirst() + .orElseThrow(); + assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY, + entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL)); } }