From ea5d315dc3aa055423d96011dfda1308125bcfdc Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Fri, 16 Feb 2024 14:58:27 +0100 Subject: [PATCH] RED-8480: updated persistence service version and added legalBasis in ManualChangeOverwrite --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../entity/ManualChangeOverwrite.java | 25 +++++---- .../server/service/ManualChangeFactory.java | 13 ++--- .../document/SectionFinderService.java | 52 +++++++++++++------ .../v1/server/RedactionIntegrationTest.java | 39 +++++++------- .../manualchanges/ManualChangesUnitTest.java | 7 ++- 7 files changed, 83 insertions(+), 57 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index de4ed8bd..77476ddd 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -7,7 +7,7 @@ description = "redaction-service-api-v1" dependencies { implementation("org.springframework:spring-web:6.0.12") - implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.338.0") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0") } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index a45fbb68..bd56e7c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.86.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.338.0" +val persistenceServiceVersion = "2.351.0" val springBootStarterVersion = "3.1.5" configurations { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java index d2e8059c..2584ff40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java @@ -27,12 +27,12 @@ import lombok.experimental.FieldDefaults; public class ManualChangeOverwrite { private static final Map, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(// - ManualRedactionEntry.class, "created by manual change", // - ManualLegalBasisChange.class, "legal basis was manually changed", // - ManualResizeRedaction.class, "resized by manual override", // - ManualForceRedaction.class, "forced by manual override", // - IdRemoval.class, "removed by manual override", // - ManualRecategorization.class, "recategorized by manual override"); + ManualRedactionEntry.class, "created by manual change", // + ManualLegalBasisChange.class, "legal basis was manually changed", // + ManualResizeRedaction.class, "resized by manual override", // + ManualForceRedaction.class, "forced by manual override", // + IdRemoval.class, "removed by manual override", // + ManualRecategorization.class, "recategorized by manual override"); List manualChanges = new LinkedList<>(); boolean changed; @@ -80,7 +80,8 @@ public class ManualChangeOverwrite { manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate)); updateFields(manualChanges); // make list unmodifiable. - return manualChanges.stream().toList(); + return manualChanges.stream() + .toList(); } @@ -121,14 +122,18 @@ public class ManualChangeOverwrite { resized = true; // This is only for not found Manual Entities. value = manualResizeRedaction.getValue(); - positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); + positions = manualResizeRedaction.getPositions() + .stream() + .map(RectangleWithPage::fromAnnotationRectangle) + .toList(); } if (manualChange instanceof ManualRecategorization recategorization) { - // recategorization logic happens in ManualChangesApplicationService. recategorized = true; - // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService. type = recategorization.getType(); + if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) { + legalBasis = recategorization.getLegalBasis(); + } } descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java index 0add07c9..d53701ce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java @@ -4,8 +4,8 @@ import java.time.OffsetDateTime; import java.util.List; import java.util.stream.Collectors; -import org.springframework.stereotype.Service; - +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -13,8 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import lombok.experimental.UtilityClass; @@ -23,7 +21,9 @@ public class ManualChangeFactory { public List toManualChangeList(List manualChanges, boolean isHint) { - return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).collect(Collectors.toList()); + return manualChanges.stream() + .map(baseAnnotation -> toManualChange(baseAnnotation, isHint)) + .collect(Collectors.toList()); } @@ -37,7 +37,8 @@ public class ManualChangeFactory { } else if (baseAnnotation instanceof ManualForceRedaction) { manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT); } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) { - manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()); + manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE) + .withChange("value", manualResizeRedact.getValue()); } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) { manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY) .withChange("value", manualRedactionEntry.getValue()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index f8f4e47c..a78360b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -57,18 +57,23 @@ public class SectionFinderService { if (entry.getContainingNodeId().isEmpty()) { continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis. } - sectionsToReanalyse.add(entry.getContainingNodeId().get(0)); + sectionsToReanalyse.add(entry.getContainingNodeId() + .get(0)); } } - var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()), - true); + var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() + .stream() + .map(DictionaryIncrementValue::getValue) + .toList(), true); - document.streamChildren().forEach(mainNode -> { - if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) { - sectionsToReanalyse.add(mainNode.getTreeId().get(0)); - } - }); + document.streamChildren() + .forEach(mainNode -> { + if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) { + sectionsToReanalyse.add(mainNode.getTreeId() + .get(0)); + } + }); Set relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest); relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds)); @@ -87,8 +92,11 @@ public class SectionFinderService { return document.getPages() .stream() .filter(page -> relevantPagesForReanalysis.contains(page.getNumber())) - .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter()))) - .map(node -> node.getTreeId().get(0)) + .flatMap(page -> Stream.concat(page.getMainBody() + .stream() + .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter()))) + .map(node -> node.getTreeId() + .get(0)) .toList(); } @@ -99,8 +107,7 @@ public class SectionFinderService { return Collections.emptySet(); } - return analyzeRequest.getManualRedactions() - .getEntriesToAdd() + return analyzeRequest.getManualRedactions().getEntriesToAdd() .stream() .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary())) .map(ManualRedactionEntry::getPositions) @@ -132,11 +139,22 @@ public class SectionFinderService { return new HashSet<>(); } - return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId), - Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId), - Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId), - Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId), - manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); + return Stream.concat(manualRedactions.getResizeRedactions() + .stream() + .map(ManualResizeRedaction::getAnnotationId), + Stream.concat(manualRedactions.getLegalBasisChanges() + .stream() + .map(ManualLegalBasisChange::getAnnotationId), + Stream.concat(manualRedactions.getRecategorizations() + .stream() + .map(ManualRecategorization::getAnnotationId), + Stream.concat(manualRedactions.getIdsToRemove() + .stream() + .map(IdRemoval::getAnnotationId), + manualRedactions.getForceRedactions() + .stream() + .map(ManualForceRedaction::getAnnotationId))))) + .collect(Collectors.toSet()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index e10a911b..95980612 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1014,7 +1014,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") .legalBasis("Something") - .requestDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) .processedDate(OffsetDateTime.now()) .build())); @@ -1040,14 +1040,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") .fileId("fileId") - .requestDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) .processedDate(OffsetDateTime.now()) .build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") .legalBasis("Manual Legal Basis Change") - .processedDate(OffsetDateTime.now()) + .processedDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now()) .build()))); manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder() @@ -1060,7 +1060,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(), Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(), Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build())) - .updateDictionary(false) + .updateDictionary(false) .build())); analyzeService.reanalyze(request); @@ -1193,7 +1193,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") .value("0049 331 441 551 14") .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) + .fileId(TEST_FILE_ID) .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") .build())) .build()); @@ -1230,14 +1230,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") .value("0049 331 441 551 14") .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) + .fileId(TEST_FILE_ID) .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") .build())) .recategorizations(Set.of(ManualRecategorization.builder() .annotationId("3029651d0842a625f2d23f8375c23600") .type("CBI_author") + .legalBasis("") .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) + .fileId(TEST_FILE_ID) .build())) .build()); @@ -1331,6 +1332,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); } + @Test @SneakyThrows public void testResizeWithUpdateDictionaryTrue() { @@ -1345,14 +1347,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get(); request.setManualRedactions(ManualRedactions.builder() - .resizeRedactions(Set.of(ManualResizeRedaction.builder() - .updateDictionary(true) - .annotationId(david.getId()) - .requestDate(OffsetDateTime.now()) - .value("David Ksenia") - .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build())) - .addToAllDossiers(false) - .build())) + .resizeRedactions(Set.of(ManualResizeRedaction.builder() + .updateDictionary(true) + .annotationId(david.getId()) + .requestDate(OffsetDateTime.now()) + .value("David Ksenia") + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build())) + .addToAllDossiers(false) + .build())) .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); @@ -1364,12 +1366,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { private IdRemoval getIdRemoval(String id) { - return IdRemoval.builder() - .annotationId(id) - .removeFromAllDossiers(false) - .removeFromDictionary(false) - .requestDate(OffsetDateTime.now()) - .build(); + return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index 85d8311c..e5761ebe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -115,7 +115,12 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())); - ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder().type("type").requestDate(start.plusSeconds(5)).annotationId(annotationId).build(); + ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder() + .type("type") + .requestDate(start.plusSeconds(5)) + .annotationId(annotationId) + .legalBasis("") + .build(); entity.getManualOverwrite().addChange(imageRecategorizationRequest); assertTrue(entity.getManualOverwrite().getRecategorized().isPresent()); assertTrue(entity.getManualOverwrite().getRecategorized().get());