From f641824270354f355dff708bf9803cf099f75727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Mon, 18 Mar 2024 13:17:11 +0100 Subject: [PATCH] RED-7384-bp: add useful fields to ManualRedactionEntry --- .../build.gradle.kts | 2 +- ...dactionLogToEntityLogMigrationService.java | 4 +- .../v1/server/model/MigrationEntity.java | 31 +-- .../document/EntityFindingUtility.java | 41 ++- .../EntityFromPrecursorCreationService.java | 13 +- .../v1/server/RedactionAcceptanceTest.java | 45 ++-- .../v1/server/RedactionIntegrationTest.java | 23 +- .../ManualChangesEnd2EndTest.java | 131 +++++---- .../ManualChangesIntegrationTest.java | 123 +++++++-- .../manualchanges/ManualChangesUnitTest.java | 66 +++-- .../manualchanges/PrecursorEntityTest.java | 51 ++-- .../UnprocessedChangesServiceTest.java | 252 +++++++++++++----- 12 files changed, 527 insertions(+), 255 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2e413162..08413ee2 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.89.3" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.349.4" +val persistenceServiceVersion = "2.349.7" val springBootStarterVersion = "3.1.5" configurations { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java index a83069a1..c9c4f996 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java @@ -34,7 +34,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; @@ -54,7 +53,6 @@ public class RedactionLogToEntityLogMigrationService { private static final double MATCH_THRESHOLD = 10; EntityFindingUtility entityFindingUtility; - EntityEnrichmentService entityEnrichmentService; DictionaryService dictionaryService; ManualChangesApplicationService manualChangesApplicationService; @@ -67,8 +65,8 @@ public class RedactionLogToEntityLogMigrationService { MigratedIds migratedIds = entitiesToMigrate.stream() .collect(new MigratedIdsCollector()); - applyManualChanges(entitiesToMigrate, manualRedactions); log.info("applying manual changes to migrated entities for file {}", fileId); + applyManualChanges(entitiesToMigrate, manualRedactions); EntityLog entityLog = new EntityLog(); entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 4761da03..5d14ad0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -4,6 +4,7 @@ import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreator import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType; import java.awt.geom.Rectangle2D; +import java.time.OffsetDateTime; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -20,6 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper; @@ -350,18 +352,8 @@ public final class MigrationEntity { manualChanges.addAll(manualChangesToApply); manualChangesToApply.forEach(manualChange -> { if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) { - // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly. - // Instead, we add an add_locally change to the db. - ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder() - .positions(manualResizeRedaction.getPositions()) - .annotationId(getNewId()) - .updateDictionary(manualResizeRedaction.getUpdateDictionary()) - .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers()) - .requestDate(manualResizeRedaction.getRequestDate()) - .textAfter(manualResizeRedaction.getTextAfter()) - .textBefore(manualResizeRedaction.getTextBefore()) - .build(); - manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction); + manualResizeRedaction.setAnnotationId(newId); + manualChangesApplicationService.resize(textEntity, manualResizeRedaction); } else { migratedEntity.getManualOverwrite().addChange(manualChange); } @@ -379,26 +371,25 @@ public final class MigrationEntity { .findFirst() .orElse(manualChanges.get(0)).getUser(); - var requestDate = manualChanges.stream() - .filter(mc -> mc instanceof ManualResizeRedaction) - .findFirst() - .orElse(manualChanges.get(0)).getRequestDate(); - + OffsetDateTime requestDate = manualChanges.get(0).getRequestDate(); return ManualRedactionEntry.builder() .annotationId(newId) .fileId(fileId) + .user(user) + .requestDate(requestDate) .type(redactionLogEntry.getType()) .value(redactionLogEntry.getValue()) .reason(redactionLogEntry.getReason()) .legalBasis(redactionLogEntry.getLegalBasis()) .section(redactionLogEntry.getSection()) - .requestDate(requestDate) + .rectangle(false) .addToDictionary(false) .addToDossierDictionary(false) - .rectangle(false) .positions(buildPositions(migratedEntity)) - .user(user) + .textAfter(redactionLogEntry.getTextAfter()) + .textBefore(redactionLogEntry.getTextBefore()) + .dictionaryEntryType(DictionaryEntryType.ENTRY) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java index e6bb27ed..77f80a0a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java @@ -47,7 +47,9 @@ public class EntityFindingUtility { } - public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map> entitiesWithSameValue, double matchThreshold) { + public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, + Map> entitiesWithSameValue, + double matchThreshold) { if (precursorEntity.getValue() == null) { return Optional.empty(); @@ -56,7 +58,7 @@ public class EntityFindingUtility { List possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH)); if (entityIdentifierValueNotFound(possibleEntities)) { - log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue()); + log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue()); return Optional.empty(); } @@ -66,18 +68,22 @@ public class EntityFindingUtility { .min(Comparator.comparingDouble(ClosestEntity::getDistance)); if (optionalClosestEntity.isEmpty()) { - log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition()); + log.info("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition()); return Optional.empty(); } ClosestEntity closestEntity = optionalClosestEntity.get(); if (closestEntity.getDistance() > matchThreshold) { - log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", - precursorEntity.getValue(), - precursorEntity.getEntityPosition().get(0).pageNumber(), - precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), - closestEntity.getDistance(), - matchThreshold); + log.info("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", + precursorEntity.getValue(), + precursorEntity.getEntityPosition() + .get(0).pageNumber(), + precursorEntity.getEntityPosition() + .stream() + .map(RectangleWithPage::rectangle2D) + .toList(), + closestEntity.getDistance(), + matchThreshold); return Optional.empty(); } @@ -93,8 +99,14 @@ public class EntityFindingUtility { private static boolean pagesMatch(TextEntity entity, List originalPositions) { - Set entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet()); - Set originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet()); + Set entityPageNumbers = entity.getPositionsOnPagePerPage() + .stream() + .map(PositionOnPage::getPage) + .map(Page::getNumber) + .collect(Collectors.toSet()); + Set originalPageNumbers = originalPositions.stream() + .map(RectangleWithPage::pageNumber) + .collect(Collectors.toSet()); return entityPageNumbers.containsAll(originalPageNumbers); } @@ -105,15 +117,16 @@ public class EntityFindingUtility { return Double.MAX_VALUE; } return originalPositions.stream() - .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())) - .average() + .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average() .orElse(Double.MAX_VALUE); } private static long countRectangles(TextEntity entity) { - return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); + return entity.getPositionsOnPagePerPage() + .stream() + .mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java index 22402160..7607a990 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java @@ -9,7 +9,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; @@ -23,29 +22,21 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service +@RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class EntityFromPrecursorCreationService { static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities EntityFindingUtility entityFindingUtility; - EntityCreationService entityCreationService; DictionaryService dictionaryService; - @Autowired - public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) { - - this.entityFindingUtility = entityFindingUtility; - entityCreationService = new EntityCreationService(entityEnrichmentService); - this.dictionaryService = dictionaryService; - } - - public List createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) { Set idRemovals = manualRedactions.getIdsToRemove(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 3f7726bc..90c9bfa1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -122,6 +122,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A."); } + @Test public void acceptanceTests() throws IOException { @@ -133,8 +134,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow(); + var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() + .orElseThrow(); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); @@ -146,8 +149,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow(); + var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst() + .orElseThrow(); assertEquals(EntryState.APPLIED, asyaLyon2.getState()); @@ -168,13 +173,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { .stream() .filter(entry -> entry.getType().equals(type)) .filter(entry -> entry.getValue().equals(value)) - .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0))); + .filter(entry -> entry.getContainingNodeId() + .get(0).equals(sectionNumber.get(0))); } private static Stream findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) { - return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); + return redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getType().equals(type)) + .filter(entry -> entry.getValue().equals(value)); } @@ -201,13 +210,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); assertEquals(EntryState.IGNORED, - findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState()); + findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)) + .findFirst() + .get().getState()); } private static IdRemoval buildIdRemoval(String id) { - return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build(); + return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 5d5869b2..da2a57ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -218,10 +218,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = ManualRedactions.builder() .resizeRedactions(Set.of(ManualResizeRedaction.builder() .annotationId("c6be5277f5ee60dc3d83527798b7fe02") + .fileId(TEST_FILE_ID) .value("Dr. Alan") .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7))) .requestDate(OffsetDateTime.now()) .updateDictionary(false) + .user("user") .build())) .build(); @@ -656,6 +658,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .fileId("fileId") .type("signature") .requestDate(OffsetDateTime.now()) + .user("user") .build())); request.setManualRedactions(manualRedactions); @@ -710,6 +713,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .annotationId("308dab9015bfafd911568cffe0a7f7de") .fileId(TEST_FILE_ID) .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC)) + .user("user") .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) .build()); @@ -718,6 +722,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .annotationId("0b56ea1a87c83f351df177315af94f0d") .fileId(TEST_FILE_ID) .legalBasis("Something") + .user("user") .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC)) .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC)) .build()); @@ -726,6 +731,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .add(IdRemoval.builder() .annotationId("0b56ea1a87c83f351df177315af94f0d") .fileId(TEST_FILE_ID) + .user("user") .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC)) .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC)) .build()); @@ -945,6 +951,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .textBefore("") .updateDictionary(false) .textAfter("") + .user("user") .build(); manualRedactions.getResizeRedactions().add(manualResizeRedaction); @@ -1032,12 +1039,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") .fileId("fileId") + .user("user") .processedDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now()) .build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") + .user("user") .legalBasis("Something") .requestDate(OffsetDateTime.now()) .processedDate(OffsetDateTime.now()) @@ -1248,6 +1257,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .requestDate(OffsetDateTime.now()) .fileId(TEST_FILE_ID) .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .user("user") .build())) .build()); @@ -1288,12 +1298,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .requestDate(OffsetDateTime.now()) .fileId(TEST_FILE_ID) .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .user("user") .build())) .recategorizations(Set.of(ManualRecategorization.builder() .annotationId("3029651d0842a625f2d23f8375c23600") .type("CBI_author") .requestDate(OffsetDateTime.now()) .fileId(TEST_FILE_ID) + .user("user") .build())) .build()); @@ -1454,6 +1466,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .resizeRedactions(Set.of(ManualResizeRedaction.builder() .updateDictionary(true) .annotationId(david.getId()) + .fileId(TEST_FILE_ID) + .user("user") .requestDate(OffsetDateTime.now()) .value("David Ksenia") .positions(List.of(Rectangle.builder() @@ -1481,7 +1495,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { private IdRemoval getIdRemoval(String id) { - return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build(); + return IdRemoval.builder() + .annotationId(id) + .removeFromAllDossiers(false) + .fileId(TEST_FILE_ID) + .user("user") + .removeFromDictionary(false) + .requestDate(OffsetDateTime.now()) + .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index c483a49a..75e0a12c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -155,29 +155,40 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String testEntityValue1 = "Desiree"; String testEntityValue2 = "Melanie"; EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); - assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); + assertEquals(2, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue1)) + .count()); + assertEquals(2, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue2)) + .count()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; - entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); + entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document) + .findFirst() + .get(); String idToResize = redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getValue().equals(testEntityValue1)) .max(Comparator.comparingInt(EntityLogEntry::getStartOffset)) - .get() - .getId(); + .get().getId(); ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder() - .annotationId(idToResize) - .value(expandedEntityKeyword) - .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), - Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) - .addToAllDossiers(false) - .updateDictionary(false) - .requestDate(OffsetDateTime.now()) - .build()); + manualRedactions.getResizeRedactions() + .add(ManualResizeRedaction.builder() + .annotationId(idToResize) + .fileId(TEST_FILE_ID) + .value(expandedEntityKeyword) + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), + Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) + .addToAllDossiers(false) + .updateDictionary(false) + .requestDate(OffsetDateTime.now()) + .build()); request.setManualRedactions(manualRedactions); analyzeService.reanalyze(request); @@ -188,21 +199,32 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { fileOutputStream.write(annotateResponse.getDocument()); } - EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); + EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(expandedEntityKeyword)) + .findFirst() + .get(); assertEquals(idToResize, resizedEntry.getId()); - assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); assertEquals(1, - redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count()); + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue1)) + .count()); + assertEquals(1, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)) + .count()); } private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(), - (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), - pageNumber); + (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } @@ -219,10 +241,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Something") - .build())); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Something") + .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); manualRedactionEntry.setAnnotationId(manualAddId); @@ -232,7 +254,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), - Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); + Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); AnalyzeRequest request = uploadFileToStorage(pdfFile); request.setManualRedactions(manualRedactions); @@ -242,11 +264,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Manual Legal Basis Change") - .requestDate(OffsetDateTime.now()) - .build()))); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Manual Legal Basis Change") + .requestDate(OffsetDateTime.now()) + .build()))); analyzeService.reanalyze(request); @@ -295,7 +317,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); - var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny(); + var optionalEntry = redactionLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findAny(); assertTrue(optionalEntry.isPresent()); assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell System.out.println("duration: " + (end - start)); @@ -345,9 +370,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); assertFalse(redactionLog2.getEntityLogEntry() - .stream() - .filter(entry -> entry.getType().equals("published_information")) - .anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); + .stream() + .filter(entry -> entry.getType().equals("published_information")) + .anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry() .stream() @@ -381,15 +406,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String annotationId = "testAnnotationId"; manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder() - .annotationId(annotationId) - .requestDate(OffsetDateTime.now()) - .type("manual") - .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") - .positions(List.of(// - new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // - new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // - new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // - .build())); + .annotationId(annotationId) + .requestDate(OffsetDateTime.now()) + .type("manual") + .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") + .positions(List.of(// + new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // + new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // + new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // + .build())); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(annotationId) .requestDate(OffsetDateTime.now()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 860f4d1b..ce91c661 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -32,18 +32,33 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualResizeRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); Set biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document) .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); - TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); + TextEntity biggerEntity = biggerEntities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(initialId) + .fileId(TEST_FILE_ID) + .user("user") .value(biggerEntity.getValue()) - .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0))) + .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage() + .get(0))) .requestDate(OffsetDateTime.now()) .updateDictionary(false) .build(); @@ -55,8 +70,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty()); assertEquals(biggerEntity.getPages(), entity.getPages()); assertEquals(biggerEntity.getValue(), entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); - assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); + assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage() + .get(0).getRectanglePerLine(), + entity.getPositionsOnPagePerPage() + .get(0).getRectanglePerLine()); assertTrue(entity.resized()); } @@ -65,12 +85,25 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualForceRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); - ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); + ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder() + .annotationId(initialId) + .fileId(TEST_FILE_ID) + .user("user") + .legalBasis("Something") + .requestDate(OffsetDateTime.now()) + .build(); doAnalysis(document, List.of(manualForceRedaction)); @@ -78,8 +111,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertFalse(entity.getIntersectingNodes().isEmpty()); assertEquals(1, entity.getPages().size()); assertEquals("David Ksenia", entity.getValue()); - assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals("Something", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertFalse(entity.removed()); assertTrue(entity.hasManualChanges()); assertTrue(entity.applied()); @@ -90,17 +127,26 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualIDRemovalTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); - IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); + IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).user("user").build(); doAnalysis(document, List.of(idRemoval)); assertEquals("David Ksenia", entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertTrue(entity.ignored()); } @@ -109,13 +155,25 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualIDRemovalButAlsoForceRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); - IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); - ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); + ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder() + .annotationId(initialId) + .legalBasis("Something") + .requestDate(OffsetDateTime.now()) + .fileId(TEST_FILE_ID) + .user("user") + .build(); doAnalysis(document, List.of(manualForceRedaction)); @@ -123,7 +181,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertFalse(entity.getIntersectingNodes().isEmpty()); assertEquals(1, entity.getPages().size()); assertEquals("David Ksenia", entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertFalse(entity.removed()); assertFalse(entity.ignored()); } @@ -131,7 +191,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { private void assertRectanglesAlmostEqual(Collection rects1, Collection rects2) { - if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { + if (rects1.stream() + .allMatch(rect1 -> rects2.stream() + .anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { return; } // use this for nice formatting of error message @@ -143,15 +205,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { double tolerance = 1e-1; return Math.abs(r1.getX() - r2.getX()) < tolerance &&// - Math.abs(r1.getY() - r2.getY()) < tolerance &&// - Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&// - Math.abs(r1.getHeight() - r2.getHeight()) < tolerance; + Math.abs(r1.getY() - r2.getY()) < tolerance &&// + Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&// + Math.abs(r1.getHeight() - r2.getHeight()) < tolerance; } private static List toAnnotationRectangles(PositionOnPage positionsOnPage) { - return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList(); + return positionsOnPage.getRectanglePerLine() + .stream() + .map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())) + .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index 85d8311c..dbf0a1c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -43,7 +43,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { OffsetDateTime start = OffsetDateTime.now(); String reason = "whatever"; Document document = buildGraphNoImages("files/new/crafted document.pdf"); - List entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList(); + List entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document) + .peek(e -> e.apply("T.0.0", reason)) + .toList(); assertFalse(entities.isEmpty()); TextEntity entity = entities.get(0); assertTrue(entity.active()); @@ -52,10 +54,11 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.resized()); assertFalse(entity.ignored()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); - String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); + String annotationId = entity.getPositionsOnPagePerPage() + .get(0).getId(); // remove first - IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); + IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).user("user").annotationId(annotationId).build(); entity.getManualOverwrite().addChange(removal); assertTrue(entity.ignored()); assertFalse(entity.applied()); @@ -65,6 +68,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { ManualForceRedaction forceRedaction = ManualForceRedaction.builder() .requestDate(start.plusSeconds(1)) .fileId(TEST_FILE_ID) + .user("user") .annotationId(annotationId) .legalBasis("coolio") .build(); @@ -73,10 +77,12 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.ignored()); assertFalse(entity.removed()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); - assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals("coolio", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); // remove again - IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build(); + IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build(); entity.getManualOverwrite().addChange(removal2); assertTrue(entity.ignored()); assertFalse(entity.applied()); @@ -86,6 +92,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { ManualForceRedaction forceRedaction2 = ManualForceRedaction.builder() .requestDate(start.plusSeconds(2)) .fileId(TEST_FILE_ID) + .user("user") .annotationId(annotationId) .legalBasis("coolio") .build(); @@ -93,7 +100,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertTrue(entity.ignored()); assertFalse(entity.applied()); assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override", - entity.buildReasonWithManualChangeDescriptions()); + entity.buildReasonWithManualChangeDescriptions()); String legalBasis = "Yeah"; String section = "Some random section!"; @@ -103,6 +110,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { .annotationId(annotationId) .requestDate(start.plusSeconds(4)) .section(section) + .fileId(TEST_FILE_ID) .user("peter") .value(value) .build(); @@ -110,16 +118,32 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertTrue(entity.ignored()); assertFalse(entity.applied()); assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed", - entity.buildReasonWithManualChangeDescriptions()); - assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue())); - assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); - assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())); + entity.buildReasonWithManualChangeDescriptions()); + assertEquals(value, + entity.getManualOverwrite().getValue() + .orElse(entity.getValue())); + assertEquals(legalBasis, + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals(section, + entity.getManualOverwrite().getSection() + .orElse(entity.getDeepestFullyContainingNode().toString())); - ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder().type("type").requestDate(start.plusSeconds(5)).annotationId(annotationId).build(); + ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder() + .type("type") + .requestDate(start.plusSeconds(5)) + .fileId(TEST_FILE_ID) + .user("user") + .annotationId(annotationId) + .build(); entity.getManualOverwrite().addChange(imageRecategorizationRequest); - assertTrue(entity.getManualOverwrite().getRecategorized().isPresent()); - assertTrue(entity.getManualOverwrite().getRecategorized().get()); - assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type())); + assertTrue(entity.getManualOverwrite().getRecategorized() + .isPresent()); + assertTrue(entity.getManualOverwrite().getRecategorized() + .get()); + assertEquals("type", + entity.getManualOverwrite().getType() + .orElse(entity.type())); } @@ -129,7 +153,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { OffsetDateTime start = OffsetDateTime.now(); String reason = "whatever"; Document document = buildGraphNoImages("files/new/crafted document.pdf"); - List entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList(); + List entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document) + .peek(e -> e.apply("T.0.0", reason)) + .toList(); assertFalse(entities.isEmpty()); TextEntity entity = entities.get(0); assertTrue(entity.active()); @@ -138,10 +164,11 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.resized()); assertFalse(entity.ignored()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); - String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); + String annotationId = entity.getPositionsOnPagePerPage() + .get(0).getId(); // remove first - IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); + IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build(); entity.getManualOverwrite().addChange(removal); assertTrue(entity.ignored()); assertFalse(entity.applied()); @@ -152,6 +179,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { .requestDate(start.plusSeconds(1)) .fileId(TEST_FILE_ID) .annotationId(annotationId) + .user("user") .legalBasis("coolio") .build(); entity.getManualOverwrite().addChange(forceRedaction); @@ -159,7 +187,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.ignored()); assertFalse(entity.removed()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); - assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals("coolio", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index b10c8a6a..b6a1efcb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -84,7 +84,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { public void testFoundManualAddRedactionAndRemovedHasStateRemoved() { DocumentAndEntity context = createFoundManualRedaction(); - IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build(); + IdRemoval removal = IdRemoval.builder().annotationId("123").user("user").fileId(TEST_FILE_ID).requestDate(OffsetDateTime.now()).build(); context.entity().getManualOverwrite().addChange(removal); assertTrue(context.entity().removed()); } @@ -95,7 +95,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() { DocumentAndEntity context = createNotFoundManualRedaction(); - IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build(); + IdRemoval removal = IdRemoval.builder().fileId(TEST_FILE_ID).user("user").annotationId("123").requestDate(OffsetDateTime.now()).build(); context.entity().getManualOverwrite().addChange(removal); assertTrue(context.entity().removed()); } @@ -108,8 +108,11 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { String value = "To: Syngenta Ltd. Jealott’s Hill"; String type = DICTIONARY_AUTHOR; ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder() + .annotationId("123") .type(type) .value(value) + .user("user") + .fileId(TEST_FILE_ID) .reason("reason") .legalBasis("n-a") .section("n-a") @@ -122,17 +125,20 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), - document, - TEST_DOSSIER_TEMPLATE_ID); + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder() + .entriesToAdd(Set.of( + manualRedactionEntry)) + .build(), + document, + TEST_DOSSIER_TEMPLATE_ID); assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); List redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(), - document, - notFoundManualEntities, - new DictionaryVersion(), - 0L).getEntityLogEntry(); + document, + notFoundManualEntities, + new DictionaryVersion(), + 0L).getEntityLogEntry(); assertEquals(1, redactionLogEntries.size()); assertEquals(value, redactionLogEntries.get(0).getValue()); @@ -146,7 +152,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/new/VV-919901.pdf"); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList(); + List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document) + .toList(); assertFalse(tempEntities.isEmpty()); var tempEntity = tempEntities.get(0); List positions = tempEntity.getPositionsOnPagePerPage() @@ -158,8 +165,11 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder() .type("manual") + .annotationId("123") .value(tempEntity.getValue()) .reason("reason") + .user("user") + .fileId(TEST_FILE_ID) .legalBasis("n-a") .section(tempEntity.getDeepestFullyContainingNode().toString()) .rectangle(false) @@ -172,21 +182,28 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { tempEntity.removeFromGraph(); assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), - document, - TEST_DOSSIER_TEMPLATE_ID); + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder() + .entriesToAdd(Set.of( + manualRedactionEntry)) + .build(), + document, + TEST_DOSSIER_TEMPLATE_ID); assertTrue(notFoundManualEntities.isEmpty()); assertEquals(1, document.getEntities().size()); - return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get()); + return new DocumentAndEntity(document, + document.getEntities() + .stream() + .findFirst() + .get()); } public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), - pageNumber); + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index 8eeba13c..d60b1c9f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -8,6 +8,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.time.OffsetDateTime; import java.util.List; import java.util.Optional; import java.util.Set; @@ -34,7 +35,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; @@ -84,6 +84,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT @SpyBean RabbitTemplate rabbitTemplate; + @BeforeEach public void stubClients() { @@ -101,21 +102,22 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } + @Test @SneakyThrows public void testManualSurroundingText() { @@ -125,10 +127,20 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(355.53775f) + .topLeftY(266.1895f) + .width(29.32224f) + .height(10.048125f) + .page(1) + .build()), + "AOEL"); var notFoundId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random"); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, + List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), + "Random"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); @@ -147,30 +159,43 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertFalse(unprocessedManualEntities.isEmpty()); assertEquals(unprocessedManualEntities.size(), 2); - Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst(); + Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream() + .filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)) + .findFirst(); assertTrue(optionalUnprocessedManualEntity.isPresent()); UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f); - assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f); - assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f); - assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).x(), 355.53775f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).y(), 266.49002f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).w(), 29.322266f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).h(), 11.017679f); - Optional optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst(); + Optional optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream() + .filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)) + .findFirst(); assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent()); UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get(); assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), ""); assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), ""); assertEquals(unprocessedNotFoundManualEntity.getSection(), ""); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getPageNumber(), 1); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[0], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[1], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[2], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[3], 1f); analyzeService.reanalyze(request); @@ -190,10 +215,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -205,13 +234,37 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); var cormsId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS"); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, + List.of(Rectangle.builder() + .topLeftX(129.86f) + .topLeftY(505.7295f) + .width(35.9904f) + .height(10.048125f) + .page(1) + .build()), + "CoRMS"); var a9Id = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G"); + ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, + List.of(Rectangle.builder() + .topLeftX(140.1096f) + .topLeftY(291.5095f) + .width(37.84512f) + .height(10.048125f) + .page(1) + .build()), + "A9396G"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); @@ -238,35 +291,53 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertFalse(unprocessedManualEntities.isEmpty()); assertEquals(unprocessedManualEntities.size(), 3); - var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst(); + var resizedAoel = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)) + .findFirst(); assertTrue(resizedAoel.isPresent()); assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); assertEquals(resizedAoel.get().getTextBefore(), "is below the "); assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(resizedAoel.get().getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(resizedAoel.get().getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(resizedAoel.get().getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(resizedAoel.get().getPositions() + .get(0).h(), positions.get(0).getHeight()); - var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst(); + var cormsResized = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)) + .findFirst(); assertTrue(cormsResized.isPresent()); assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of"); assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore "); assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the"); - assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX()); - assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY()); - assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth()); - assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight()); + assertEquals(cormsResized.get().getPositions() + .get(0).x(), positions2.get(0).getTopLeftX()); + assertEquals(cormsResized.get().getPositions() + .get(0).y(), positions2.get(0).getTopLeftY()); + assertEquals(cormsResized.get().getPositions() + .get(0).w(), positions2.get(0).getWidth()); + assertEquals(cormsResized.get().getPositions() + .get(0).h(), positions2.get(0).getHeight()); - var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst(); + var a9Resized = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)) + .findFirst(); assertTrue(a9Resized.isPresent()); assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); assertEquals(a9Resized.get().getTextBefore(), "data for S"); assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX()); - assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY()); - assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth()); - assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight()); + assertEquals(a9Resized.get().getPositions() + .get(0).x(), positions3.get(0).getTopLeftX()); + assertEquals(a9Resized.get().getPositions() + .get(0).y(), positions3.get(0).getTopLeftY()); + assertEquals(a9Resized.get().getPositions() + .get(0).w(), positions3.get(0).getWidth()); + assertEquals(a9Resized.get().getPositions() + .get(0).h(), positions3.get(0).getHeight()); } @@ -277,7 +348,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -301,10 +380,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -315,7 +398,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -339,10 +430,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -353,7 +448,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -377,25 +480,32 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT private static ManualResizeRedaction prepareManualSizeRedaction(String id, List positions, String value) { - ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction(); - manualResizeRedaction.setAnnotationId(id); - manualResizeRedaction.setPositions(positions); - manualResizeRedaction.setUpdateDictionary(false); - manualResizeRedaction.setAddToAllDossiers(false); - manualResizeRedaction.setValue(value); - return manualResizeRedaction; + return ManualResizeRedaction.builder() + .annotationId(id) + .fileId("fileId") + .user("user") + .positions(positions) + .updateDictionary(false) + .addToAllDossiers(false) + .value(value) + .requestDate(OffsetDateTime.now()) + .build(); } private static ManualRedactionEntry prepareManualRedactionEntry(String id, List positions, String value) { - ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); - manualRedactionEntry.setAnnotationId(id); - manualRedactionEntry.setFileId("fileId"); - manualRedactionEntry.setType("CBI_author"); - manualRedactionEntry.setValue(value); - manualRedactionEntry.setReason("Manual Redaction"); - manualRedactionEntry.setPositions(positions); - return manualRedactionEntry; + return ManualRedactionEntry.builder() + .annotationId(id) + .fileId("fileId") + .user("user") + .type("CBI_author") + .value(value) + .reason("Manual Redaction") + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .positions(positions) + .build(); } + }