diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 5619255d..726dcf28 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.71.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.207.0" +val persistenceServiceVersion = "2.210.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 6abf510e..48fa20cc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -71,7 +71,6 @@ public class AnalyzeService { ComponentLogCreatorService componentLogCreatorService; RedactionStorageService redactionStorageService; RedactionChangeLogService redactionChangeLogService; - EntityChangeLogService entityChangeLogService; LegalBasisClient legalBasisClient; RedactionServiceSettings redactionServiceSettings; ImportedRedactionService importedRedactionService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index 039b9290..64397765 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -26,9 +26,14 @@ public class EntityChangeLogService { @Timed("redactmanager_computeChanges") public boolean computeChanges(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber) { + var now = OffsetDateTime.now(); + if (previousEntityLogEntries.isEmpty()) { + newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now))); + return true; + } + boolean hasChanges = false; - var now = OffsetDateTime.now(); for (EntityLogEntry entityLogEntry : newEntityLogEntries) { Optional optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny(); if (optionalPreviousEntity.isEmpty()) { @@ -46,11 +51,21 @@ public class EntityChangeLogService { } } } + addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now); + return hasChanges; + } + + + private static void addRemovedEntriesAsRemoved(List previousEntityLogEntries, + List newEntityLogEntries, + int analysisNumber, + OffsetDateTime now) { + Set existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); List removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList(); removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); - return hasChanges; + newEntityLogEntries.addAll(removedEntries); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index e915cc0f..8a2bbfcf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -12,8 +11,6 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; @@ -33,21 +30,26 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Service @Slf4j @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class EntityLogCreatorService { - private final DictionaryService dictionaryService; - private final ManualChangeFactory manualChangeFactory; - private final ImportedRedactionService importedRedactionService; - private final RedactionServiceSettings redactionServiceSettings; - private final LegalBasisClient legalBasisClient; - private final EntityChangeLogService entityChangeLogService; + DictionaryService dictionaryService; + ManualChangeFactory manualChangeFactory; + ImportedRedactionService importedRedactionService; + RedactionServiceSettings redactionServiceSettings; + LegalBasisClient legalBasisClient; + EntityChangeLogService entityChangeLogService; + RedactionStorageService redactionStorageService; private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) { @@ -65,7 +67,6 @@ public class EntityLogCreatorService { List entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), entityLogEntries, @@ -82,13 +83,23 @@ public class EntityLogCreatorService { true); entityLog.setEntityLogEntry(importedRedactionFilteredEntries); - var now = OffsetDateTime.now(); - entityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analyzeRequest.getAnalysisNumber(), ChangeType.ADDED, now))); + List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages()); return entityLog; } + private List getPreviousEntityLogEntries(String dossierId, String fileId) { + + if (redactionStorageService.entityLogExists(dossierId, fileId)) { + return redactionStorageService.getEntityLog(dossierId, fileId).getEntityLogEntry(); + } else { + return Collections.emptyList(); + } + } + + public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) { List legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId); @@ -207,8 +218,7 @@ public class EntityLogCreatorService { boolean isHint = isHint(manualEntity.getEntityType()); return EntityLogEntry.builder().id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()).type(type).state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity)) + .legalBasis(manualEntity.legalBasis()).value(manualEntity.value()).type(type).state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity)) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .containingNodeId(Collections.emptyList()) .closestHeadline("") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java index 19574ce6..0fed206c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; @@ -58,7 +59,7 @@ public class ImportedRedactionService { List redactionLogEntries, boolean addImportedRedactions) { - var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId); + ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId); if (importedRedactions == null) { return redactionLogEntries; } @@ -103,7 +104,7 @@ public class ImportedRedactionService { for (var importedRedaction : importedRedactionsValues) { EntityLogEntry redactionLogEntry = EntityLogEntry.builder() .id(importedRedaction.getId()) - .type(IMPORTED_REDACTION_TYPE) + .type(IMPORTED_REDACTION_TYPE).entryType(EntryType.ENTITY) .imported(true) .state(EntryState.APPLIED) .positions(toPositions(importedRedaction.getPositions())) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java index d2c26f5f..cf601835 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java @@ -22,6 +22,7 @@ import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +@Deprecated(forRemoval = true) @Slf4j @Service @RequiredArgsConstructor diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 79340cd5..35d3f639 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +@Deprecated(forRemoval = true) @Service @Slf4j @RequiredArgsConstructor @@ -204,7 +205,6 @@ public class RedactionLogCreatorService { }); manualChange1.setProcessedDate(manualChange.getProcessedDate()); manualChange1.setRequestedDate(manualChange.getRequestedDate()); - manualChange1.setAnnotationStatus(manualChange.getAnnotationStatus()); manualChange1.setPropertyChanges(manualChange.getPropertyChanges()); return manualChange1; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 8f4ddbd1..9b4d8c77 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -86,6 +86,7 @@ public class RedactionStorageService { } + @Deprecated(forRemoval = true) @Timed("redactmanager_getRedactionLog") public RedactionLog getRedactionLog(String dossierId, String fileId) { @@ -160,6 +161,12 @@ public class RedactionStorageService { } + public boolean entityLogExists(String dossierId, String fileId) { + + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)); + } + + @RequiredArgsConstructor public enum StorageType { PARSED_DOCUMENT(".json"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java index b0880c33..41edfba8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java @@ -12,7 +12,7 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import lombok.AllArgsConstructor; @@ -26,18 +26,9 @@ public class RectangleTransformations { } - public static Rectangle2D rectangleBBox(List rectangles) { + public static Rectangle2D rectangleBBox(List positions) { - return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector()); - } - - - public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) { - - return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(), - redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(), - redactionLogRectangle.getWidth(), - -redactionLogRectangle.getHeight()); + return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 237f58a8..1044b943 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -48,7 +48,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { // @Disabled public void titleExtraction() throws IOException { - AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/A8591B/15-Curacron_ToxicidadeAgudaOral.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf"); // AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf", // "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json"); @@ -58,7 +58,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -84,7 +84,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -104,14 +104,14 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { // Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf // TODO unify logic - AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request); System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 79933126..3ed2941b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -44,6 +45,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; @@ -59,7 +63,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; @@ -161,11 +164,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - Map> duplicates = new HashMap<>(); + Map> duplicates = new HashMap<>(); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - redactionLog.getRedactionLogEntry().forEach(entry -> { + entityLog.getEntityLogEntry().forEach(entry -> { duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); }); @@ -202,9 +205,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList()); + var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname"); } @@ -213,13 +216,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -257,7 +260,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var toRemove = IdRemoval.builder() .annotationId("c630599611e6e3db314518374bcf70f7") @@ -272,13 +275,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { request.setManualRedactions(manualRedactions); analyzeService.reanalyze(request); - var mergedRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); - assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse(); + var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse(); - var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); - assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue(); + var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue(); } @@ -303,11 +306,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart)); - Map> duplicates = new HashMap<>(); + Map> duplicates = new HashMap<>(); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - redactionLog.getRedactionLogEntry().forEach(entry -> { + entityLog.getEntityLogEntry().forEach(entry -> { duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); }); @@ -352,7 +355,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); long end = System.currentTimeMillis(); @@ -365,15 +368,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { int correctFound = 0; loop: - for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) { + for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) { for (Section section : documentGraph.getMainSections()) { - if (redactionLogEntry.isImage()) { + if (entityLogEntry.getEntryType().equals(EntryType.IMAGE)) { correctFound++; continue loop; } - if (redactionLogEntry.getSectionNumber() == section.getTreeId().get(0)) { - String value = section.getTextBlock().subSequence(new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString(); - if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { + if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) { + String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString(); + if (entityLogEntry.getValue().equalsIgnoreCase(value)) { correctFound++; } else { throw new RuntimeException("WTF"); @@ -414,7 +417,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); @@ -434,7 +437,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); System.out.println("Output file:" + outputFileName); } @@ -468,9 +471,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { mockDictionaryCalls(3L); analyzeService.reanalyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges(); + var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges(); assertThat(changes.size()).isEqualTo(2); @@ -507,7 +510,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); long end = System.currentTimeMillis(); @@ -518,13 +521,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID))); } - List valuesInDocument = redactionLog.getRedactionLogEntry() - .stream() - .filter(e -> !e.isImage()) + List valuesInDocument = entityLog.getEntityLogEntry() + .stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)) .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())) .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString()) .toList(); - List valuesInRedactionLog = redactionLog.getRedactionLogEntry().stream().filter(e -> !e.isImage()).map(RedactionLogEntry::getValue).toList(); + List valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList(); assertEquals(valuesInRedactionLog, valuesInDocument); @@ -562,7 +564,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); @@ -583,7 +585,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); System.out.println("Output file:" + outputFileName); } @@ -810,8 +812,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(5); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(5); long end = System.currentTimeMillis(); @@ -857,7 +859,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() .manualRedactions(manualRedactions) @@ -885,10 +887,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - redactionLog.getRedactionLogEntry().forEach(entry -> { - if (!entry.isHint()) { + redactionLog.getEntityLogEntry().forEach(entry -> { + if (!entry.getEntryType().equals(EntryType.HINT)) { assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); } }); @@ -1048,7 +1050,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -1056,7 +1058,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - redactionLog.getRedactionLogEntry().forEach(entry -> { + entityLog.getEntityLogEntry().forEach(entry -> { if (entry.getValue() == null) { return; } @@ -1094,8 +1096,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList()); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); assertThat(values).contains("Mrs. Robinson"); assertThat(values).contains("Mr. Bojangles"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 45909ba4..5aff0602 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -22,9 +22,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -120,22 +122,20 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1); + assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(1); - RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); + EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry().get(0); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary); - assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); - assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false); - assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false); + assertThat(redactionLogEntry.getState()).isEqualTo(EntryState.APPLIED); assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); - assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true); + assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } @@ -158,22 +158,20 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1); + assertThat(redactionLog.getEntityLogEntry().size()).isEqualTo(1); - RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); + EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry().get(0); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary); - assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); - assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false); - assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false); + assertThat(redactionLogEntry.getState()).isEqualTo(EntryState.APPLIED); assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); - assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true); + assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 9a6545f8..b57bfbc0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -508,7 +508,6 @@ public class RulesTest { var savedManualChange = savedRedactionLogEntry.get() .getManualChanges() .stream() - .filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus()) .filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name())) .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId())) .filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 98875593..036814f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -1,11 +1,9 @@ package com.iqser.red.service.redaction.v1.server.annotate; -import java.awt.Color; import java.awt.geom.Rectangle2D; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -15,32 +13,26 @@ import java.util.stream.Stream; import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.PDPageContentStream; import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.apache.pdfbox.pdmodel.font.PDType1Font; -import org.apache.pdfbox.pdmodel.font.Standard14Fonts; import org.apache.pdfbox.pdmodel.graphics.color.PDColor; import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight; -import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText; import org.springframework.stereotype.Service; import com.google.common.primitives.Floats; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; -import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; @Service @RequiredArgsConstructor @@ -64,46 +56,38 @@ public class AnnotationService { * @param annotateRequest containing the SectionGrid, RedactionLog, ManualRedactions and the colors for the highlights * @return the annotated PDFDoc */ + @SneakyThrows public AnnotateResponse annotate(AnnotateRequest annotateRequest) { var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), - annotateRequest.getFileId(), - FileType.ORIGIN)); + annotateRequest.getFileId(), FileType.VIEWER_DOCUMENT)); - var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId()); - var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId()); + var entityLog = redactionStorageService.getEntityLog(annotateRequest.getDossierId(), annotateRequest.getFileId()); try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) { pdDocument.setAllSecurityToBeRemoved(true); dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId()); - annotate(pdDocument, redactionLog, sectionsGrid); + annotate(pdDocument, entityLog); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { pdDocument.save(byteArrayOutputStream); return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build(); } - } catch (Exception e) { - throw new RedactionException(e); } } - private void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException { + private void annotate(PDDocument document, EntityLog entityLog) throws IOException { - Map> redactionLogPerPage = convertRedactionLog(redactionLog); + Map> redactionLogPerPage = groupingByPageNumber(entityLog); for (int page = 1; page <= document.getNumberOfPages(); page++) { PDPage pdPage = document.getPage(page - 1); - List sectionRectangles = sectionGrid.getRectanglesPerPage().get(page); - if (sectionRectangles != null && !sectionRectangles.isEmpty()) { - drawSectionGrid(document, pdPage, sectionRectangles); - } - - List logEntries = redactionLogPerPage.get(page); + List logEntries = redactionLogPerPage.get(page); if (logEntries != null && !logEntries.isEmpty()) { addAnnotations(logEntries, pdPage, page); } @@ -111,19 +95,67 @@ public class AnnotationService { } - private void addAnnotations(List logEntries, PDPage pdPage, int page) throws IOException { + private Map> groupingByPageNumber(EntityLog redactionLog) { + + Map> redactionLogPerPage = new HashMap<>(); + if (redactionLog == null) { + return redactionLogPerPage; + } + for (EntityLogEntry entry : redactionLog.getEntityLogEntry()) { + int page = 0; + for (Position position : entry.getPositions()) { + if (position.getPageNumber() != page) { + redactionLogPerPage.computeIfAbsent(position.getPageNumber(), x -> new ArrayList<>()).add(entry); + page = position.getPageNumber(); + } + } + } + return redactionLogPerPage; + } + + + private void addAnnotations(List logEntries, PDPage pdPage, int page) throws IOException { List annotations = pdPage.getAnnotations(); - for (RedactionLogEntry entry : logEntries) { - if (entry.lastChangeIsRemoved()) { + for (EntityLogEntry entry : logEntries) { + if (entry.getState().equals(EntryState.REMOVED)) { continue; } annotations.addAll(createAnnotation(entry, page, pdPage.getRotation(), pdPage.getCropBox())); } } - public static PDRectangle toPDRectangleBBox(List rectangles) { + + private List createAnnotation(EntityLogEntry redactionLogEntry, int page, int rotation, PDRectangle cropBox) { + + List annotations = new ArrayList<>(); + + List rectangles = redactionLogEntry.getPositions().stream().filter(pos -> pos.getPageNumber() == page).collect(Collectors.toList()); + + if (rectangles.isEmpty()) { + return annotations; + } + + PDAnnotationHighlight annotation = new PDAnnotationHighlight(); + annotation.constructAppearances(); + PDRectangle pdRectangle = toPDRectangleBBox(rectangles); + annotation.setRectangle(pdRectangle); + annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles))); + if (!(redactionLogEntry.getEntryType().equals(EntryType.HINT) || redactionLogEntry.getState().equals(EntryState.IGNORED))) { + annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry)); + } + annotation.setTitlePopup(redactionLogEntry.getId()); + annotation.setAnnotationName(redactionLogEntry.getId()); + annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE)); + annotation.setNoRotate(false); + annotations.add(annotation); + + return annotations; + } + + + public static PDRectangle toPDRectangleBBox(List rectangles) { Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles); @@ -135,69 +167,12 @@ public class AnnotationService { return annotationPosition; } - private List createAnnotation(RedactionLogEntry redactionLogEntry, int page, int rotation, PDRectangle cropBox) { - List annotations = new ArrayList<>(); + public static List toQuadPoints(List rectangles) { - List rectangles = redactionLogEntry.getPositions().stream().filter(pos -> pos.getPage() == page).collect(Collectors.toList()); - - if (rectangles.isEmpty()) { - return annotations; - } - - PDAnnotationHighlight annotation = new PDAnnotationHighlight(); - annotation.constructAppearances(); - PDRectangle pdRectangle = toPDRectangleBBox(rectangles); - annotation.setRectangle(pdRectangle); - annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles))); - if (!redactionLogEntry.isHint()) { - annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry)); - } - annotation.setTitlePopup(redactionLogEntry.getId()); - annotation.setAnnotationName(redactionLogEntry.getId()); - annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE)); - annotation.setNoRotate(false); - annotations.add(annotation); - - if (redactionLogEntry.getComments() != null) { - for (RedactionLogComment comment : redactionLogEntry.getComments()) { - PDAnnotationText txtAnnot = new PDAnnotationText(); - txtAnnot.setAnnotationName(String.valueOf(comment.getId())); - txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation - txtAnnot.setName(PDAnnotationText.NAME_COMMENT); - txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime())); - txtAnnot.setTitlePopup(comment.getUser()); - txtAnnot.setContents(comment.getText()); - txtAnnot.setRectangle(pdRectangle); - annotations.add(txtAnnot); - } - } - - return annotations; + return rectangles.stream().map(Position::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList(); } - - private String createAnnotationContent(RedactionLogEntry redactionLogEntry) { - - return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\""; - } - - - public static List toQuadPoints(List rectangles) { - - return rectangles.stream().map(AnnotationService::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList(); - } - - - private static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) { - - return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(), - redactionLogRectangle.getTopLeft().getY(), - redactionLogRectangle.getWidth(), - redactionLogRectangle.getHeight()); - } - - public static Stream toQuadPoints(Rectangle2D rectangle) { double x1 = rectangle.getMinX(); @@ -215,71 +190,9 @@ public class AnnotationService { } - private void drawSectionGrid(PDDocument document, PDPage pdPage, List sectionRectangles) throws IOException { + private String createAnnotationContent(EntityLogEntry redactionLogEntry) { - PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true); - for (SectionRectangle sectionRectangle : sectionRectangles) { - drawSectionRectangle(contentStream, sectionRectangle); - drawSectionPartNumberText(contentStream, sectionRectangle); - drawTableCells(contentStream, sectionRectangle); - } - contentStream.close(); - } - - - private void drawSectionRectangle(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException { - - contentStream.setStrokingColor(Color.LIGHT_GRAY); - contentStream.setLineWidth(0.5f); - contentStream.addRect(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY(), sectionRectangle.getWidth(), sectionRectangle.getHeight()); - contentStream.stroke(); - } - - - private void drawSectionPartNumberText(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException { - - contentStream.beginText(); - contentStream.setNonStrokingColor(Color.DARK_GRAY); - contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.TIMES_ROMAN), 8f); - if (sectionRectangle.getTableCells() == null) { - contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY() + sectionRectangle.getHeight()); - } else { - contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY()); - } - contentStream.showText(sectionRectangle.getPart() + "/" + sectionRectangle.getNumberOfParts()); - contentStream.endText(); - } - - - private void drawTableCells(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException { - - if (sectionRectangle.getTableCells() != null) { - for (CellRectangle cell : sectionRectangle.getTableCells()) { - contentStream.setLineWidth(0.5f); - contentStream.setStrokingColor(Color.CYAN); - contentStream.addRect(cell.getTopLeft().getX(), cell.getTopLeft().getY(), cell.getWidth(), cell.getHeight()); - contentStream.stroke(); - } - } - } - - - private Map> convertRedactionLog(RedactionLog redactionLog) { - - Map> redactionLogPerPage = new HashMap<>(); - if (redactionLog == null) { - return redactionLogPerPage; - } - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - int page = 0; - for (Rectangle position : entry.getPositions()) { - if (position.getPage() != page) { - redactionLogPerPage.computeIfAbsent(position.getPage(), x -> new ArrayList<>()).add(entry); - page = position.getPage(); - } - } - } - return redactionLogPerPage; + return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\""; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 04cc9b95..bddd9ac3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -2,8 +2,8 @@ package com.iqser.red.service.redaction.v1.server.manualchanges; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.when; -import static org.wildfly.common.Assert.assertTrue; import java.awt.geom.Rectangle2D; import java.io.File; @@ -34,6 +34,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; @@ -46,9 +49,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; @@ -148,18 +148,17 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String testEntityValue1 = "Desiree"; String testEntityValue2 = "Melanie"; - RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); - assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); + EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); + assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); - String idToResize = redactionLog.getRedactionLogEntry() + String idToResize = redactionLog.getEntityLogEntry() .stream() - .filter(entry -> entry.getValue().equals(testEntityValue1)) - .max(Comparator.comparingInt(RedactionLogEntry::getStartOffset)) + .filter(entry -> entry.getValue().equals(testEntityValue1)).max(Comparator.comparingInt(EntityLogEntry::getStartOffset)) .get() .getId(); List resizedPositions = expandedEntity.getPositionsOnPagePerPage() @@ -181,18 +180,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { request.setManualRedactions(manualRedactions); AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); - redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf"); File tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile(); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { fileOutputStream.write(annotateResponse.getDocument()); } - RedactionLogEntry resizedEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); - assertTrue(resizedEntry.getChanges().get(resizedEntry.getChanges().size() - 1).getType().equals(ChangeType.CHANGED)); + EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); assertEquals(idToResize, resizedEntry.getId()); - assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); - assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.lastChangeIsRemoved()).count()); + assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); + assertEquals(1, + redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count()); } @@ -257,7 +256,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -278,16 +277,16 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { AnalyzeRequest request = uploadFileToStorage(filePath); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); - RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var oxfordUniversityPress = redactionLog.getRedactionLogEntry() + var oxfordUniversityPress = redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals("published_information")) .filter(entry -> entry.getValue().equals("Oxford University Press")) .findFirst() .get(); - var asyaLyon = redactionLog.getRedactionLogEntry() + var asyaLyon = redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals("CBI_author")) .filter(entry -> entry.getValue().equals("Asya Lyon")) @@ -296,7 +295,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { assertEquals("CBI.7.0", asyaLyon.getMatchedRule()); assertEquals("Published Information found in section", asyaLyon.getReason()); - assertFalse(asyaLyon.isRedacted()); + assertNotEquals(asyaLyon.getState(), EntryState.APPLIED); ManualRecategorization recategorization = ManualRecategorization.builder() .requestDate(OffsetDateTime.now()) @@ -311,27 +310,27 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { analyzeService.reanalyze(request); - RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertFalse(redactionLog2.getRedactionLogEntry() + EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertFalse(redactionLog2.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals("published_information")) .anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); - var oxfordUniversityPressRecategorized = redactionLog2.getRedactionLogEntry() + var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals("vertebrate")) .filter(entry -> entry.getValue().equals("Oxford University Press")) .findFirst() .get(); - var asyaLyon2 = redactionLog2.getRedactionLogEntry() + var asyaLyon2 = redactionLog2.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals("CBI_author")) .filter(entry -> entry.getValue().equals("Asya Lyon")) .findFirst() .get(); - assertTrue(asyaLyon2.isRedacted()); + assertEquals(asyaLyon2.getState(), EntryState.APPLIED); assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java index 4366b958..7441504e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java @@ -7,7 +7,6 @@ import static org.wildfly.common.Assert.assertFalse; import static org.wildfly.common.Assert.assertTrue; import java.awt.geom.Rectangle2D; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -17,20 +16,22 @@ import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.service.DictionaryService; -import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService; import lombok.SneakyThrows; @@ -43,7 +44,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes private ManualEntityCreationService manualEntityCreationService; @Autowired - private RedactionLogCreatorService redactionLogCreatorService; + private EntityLogCreatorService entityLogCreatorService; @MockBean private DictionaryService dictionaryService; @@ -126,10 +127,11 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); - List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - TEST_DOSSIER_TEMPLATE_ID, + List redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(), + document, notFoundManualEntities, - Collections.emptyMap()); + new DictionaryVersion(), + 0L).getEntityLogEntry(); assertEquals(1, redactionLogEntries.size()); assertEquals(value, redactionLogEntries.get(0).getValue()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt index 1a480892..cbcbfe3d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt @@ -29,4 +29,4 @@ J.B. RASCLE 青森植 サンプル量 供試試料 （無処理 区） -材料 +材料 \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII_false_positive.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII_false_positive.txt index cd9fab48..492ca143 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII_false_positive.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII_false_positive.txt @@ -1 +1 @@ -C. J. Alfred Xinyi \ No newline at end of file +C. J. Alfred XinyiP \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study-ocred.pdf similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study.pdf rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study-ocred.pdf index 289b46cf..6ad1b53f 100644 Binary files a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study.pdf and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study-ocred.pdf differ