From ac38a966c5dbc85b1524e9a3b20db8271bb87f0a Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Fri, 23 Feb 2024 13:22:53 +0100 Subject: [PATCH] RED-8589 - Add "MANUAL" engine to all annotations that has entries in... --- .../service/EntityLogCreatorService.java | 35 ++++++++---- .../ManualChangesEnd2EndTest.java | 56 ++++++++++++++++++- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index f630a968..6cd69627 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; @@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; @@ -213,11 +215,13 @@ public class EntityLogCreatorService { .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) .containingNodeId(image.getTreeId()) .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) - .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) + .section(image.getManualOverwrite().getSection() + .orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) .state(buildEntryState(image)) .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE) + .engines(getEngines(null, image.getManualOverwrite())) .build(); } @@ -245,13 +249,12 @@ public class EntityLogCreatorService { .textBefore("") .startOffset(-1) .endOffset(-1) - .positions(precursorEntity.getManualOverwrite() - .getPositions() - .orElse(precursorEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) - .engines(precursorEntity.getEngines()) + .positions(precursorEntity.getManualOverwrite().getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) + .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite())) //imported is no longer used, frontend should check engines //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) .imported(false) @@ -271,7 +274,8 @@ public class EntityLogCreatorService { .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.type()) - .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) + .section(entity.getManualOverwrite().getSection() + .orElse(entity.getDeepestFullyContainingNode().toString())) .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) @@ -281,7 +285,7 @@ public class EntityLogCreatorService { .startOffset(entity.getTextRange().start()) .endOffset(entity.getTextRange().end()) .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + .engines(getEngines(entity.getEngines(), entity.getManualOverwrite())) //imported is no longer used, frontend should check engines //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) .imported(false) @@ -293,6 +297,17 @@ public class EntityLogCreatorService { } + private Set getEngines(Set currentEngines, ManualChangeOverwrite manualChangeOverwrite) { + + Set engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>(); + + if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) { + engines.add(Engine.MANUAL); + } + return engines; + } + + private boolean isHint(EntityType entityType) { return entityType.equals(EntityType.HINT); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 128849f7..c483a49a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; @@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; @@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .filter(entry -> entry.getValue().equals("Oxford University Press")) .findFirst() .get(); + assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL)); var asyaLyon = redactionLog.getEntityLogEntry() .stream() @@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { assertEquals(asyaLyon2.getState(), EntryState.APPLIED); assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size()); + assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL)); } @@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); - EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow(); + EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getId().equals(annotationId)) + .findFirst() + .orElseThrow(); assertEquals("Expand to Hint", entityLogEntry.getValue()); assertEquals(1, entityLogEntry.getPositions().size()); - assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertEquals(ManualRedactionType.RESIZE, + entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL)); + } + + + @Test + @SneakyThrows + public void testAddEngineManualToResizeDictionaryEntry() { + + String filePath = "files/new/crafted document.pdf"; + AnalyzeRequest request = uploadFileToStorage(filePath); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + AnalyzeResult result = analyzeService.analyze(request); + ManualRedactions manualRedactions = new ManualRedactions(); + + EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); + var dictionaryEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()) + .findFirst() + .get(); + ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() + .annotationId(dictionaryEntry.getId()) + .requestDate(OffsetDateTime.now()) + .value("Image") + .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1))) + .updateDictionary(true) + .build(); + manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction)); + request.setManualRedactions(manualRedactions); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId()); + EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getId().equals(dictionaryEntry.getId())) + .findFirst() + .orElseThrow(); + assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY, + entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType()); + assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL)); } }