From 13363acc8365d2dde7bd75184d38dec41a6713ab Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 10 Jun 2024 13:22:08 +0200 Subject: [PATCH] RED-9246: Resize on dossier level for dossier-template-level entry becomes skipped * added specific unit test for this case --- .../server/model/dictionary/Dictionary.java | 3 + .../AbstractRedactionIntegrationTest.java | 69 ++++--- .../redaction/v1/server/AnalysisTest.java | 14 +- .../v1/server/DocumineFloraTest.java | 13 +- .../v1/server/MigrationIntegrationTest.java | 13 +- .../v1/server/RedactionAcceptanceTest.java | 14 +- .../v1/server/RedactionIntegrationTest.java | 171 +++++++++++++++--- .../v1/server/RedactionIntegrationV2Test.java | 13 +- .../redaction/v1/server/RulesTest.java | 52 +++--- .../DocumentPerformanceIntegrationTest.java | 13 +- .../ManualChangesEnd2EndTest.java | 15 +- .../UnprocessedChangesServiceTest.java | 14 +- 12 files changed, 240 insertions(+), 164 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 472a610f..a650d374 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -31,6 +31,9 @@ public class Dictionary { @Getter private List dictionaryModels; + // todo: dossier and dossier template level DictionaryModels override each other + // at the moment there are no problems because they always have the same rank / hint information + // but it should be changed so that the localAccessMap contains all models private Map localAccessMap = new HashMap<>(); @Getter diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index e116014b..fa6ae941 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -100,8 +100,10 @@ public abstract class AbstractRedactionIntegrationTest { protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple"; protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; - public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; - public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + protected final static String TEST_DOSSIER_ID = "123"; + public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; + public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; + public static final String DOSSIER_AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String FORMULA_TYPE_ID = FORMULA_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String SIGNATURE_TYPE_ID = SIGNATURE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -179,9 +181,9 @@ public abstract class AbstractRedactionIntegrationTest { protected final Map rankTypeMap = new HashMap<>(); protected final Colors colors = new Colors(); protected final Map reanlysisVersions = new HashMap<>(); - protected final Set deleted = new HashSet<>(); + protected final Set deletedInTemplate = new HashSet<>(); + protected final Set deletedInDossier = new HashSet<>(); - protected final static String TEST_DOSSIER_ID = "123"; protected final static String TEST_FILE_ID = "123"; @MockBean @@ -249,6 +251,7 @@ public abstract class AbstractRedactionIntegrationTest { true)); when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR, true)); + when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true)); } @@ -346,6 +349,7 @@ public abstract class AbstractRedactionIntegrationTest { .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); + dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>()); falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") @@ -399,6 +403,7 @@ public abstract class AbstractRedactionIntegrationTest { typeColorMap.put(FORMULA_INDICATOR, "#ffe187"); typeColorMap.put(SIGNATURE_INDICATOR, "#ffe187"); typeColorMap.put(IMPORTED_REDACTION_INDICATOR, "#fcfbe6"); + typeColorMap.put(DOSSIER_REDACTIONS_INDICATOR, "#ffe187"); typeColorMap.put(ROTATE_SIMPLE_INDICATOR, "#66ccff"); hintTypeMap.put(VERTEBRATE_INDICATOR, true); @@ -501,33 +506,54 @@ public abstract class AbstractRedactionIntegrationTest { } - protected List getTypeResponse() { + protected List getTemplateDictionaryTypeResponse() { - return typeColorMap.entrySet() + return dictionary.keySet() .stream() - .map(typeColor -> Type.builder() - .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(typeColor.getKey()) + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(key) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor(typeColor.getValue()) - .isHint(hintTypeMap.get(typeColor.getKey())) - .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) - .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) - .rank(rankTypeMap.get(typeColor.getKey())) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) .build()) + .collect(Collectors.toList()); + } + + + protected List getDossierDictionaryTypeResponse() { + + return dossierDictionary.keySet() + .stream() + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID) + .type(key) + .dossierId(TEST_DOSSIER_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) + .build()) .collect(Collectors.toList()); } protected Type getDictionaryResponse(String type, boolean isDossierDictionary) { + String id = type + ":" + TEST_DOSSIER_TEMPLATE_ID; + return Type.builder() - .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID) + .id(isDossierDictionary ? id + ":" + TEST_DOSSIER_ID : id) .hexColor(typeColorMap.get(type)) - .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) - .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>()) - .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>()) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type), true) : toDictionaryEntry(dictionary.get(type), false)) + .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type), isDossierDictionary) : new ArrayList<>()) + .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type), isDossierDictionary) : new ArrayList<>()) .isHint(hintTypeMap.get(type)) .isCaseInsensitive(caseInSensitiveMap.get(type)) .isRecommendation(recommendationTypeMap.get(type)) @@ -542,20 +568,21 @@ public abstract class AbstractRedactionIntegrationTest { } - private List toDictionaryEntry(List entries) { + private List toDictionaryEntry(List entries, boolean isDossierDictionary) { if (entries == null) { return Collections.emptyList(); } return entries.stream() - .map(this::toDictionaryEntry) + .map(entry -> toDictionaryEntry(entry, isDossierDictionary)) .collect(Collectors.toList()); } - private DictionaryEntry toDictionaryEntry(String entry) { + private DictionaryEntry toDictionaryEntry(String entry, boolean isDossierDictionary) { + var deleted = isDossierDictionary ? deletedInDossier : deletedInTemplate; return DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java index 1e3bda77..83d249d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.mockito.Mockito.when; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.util.List; @@ -100,19 +99,10 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 739c6c30..bb82c5a9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -209,19 +209,10 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java index a08e09b4..2821f125 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java @@ -83,19 +83,10 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index fb314ba9..88e74cd0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -83,20 +83,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); - + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 5a1cd462..d0922fab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -27,7 +27,10 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -84,6 +87,7 @@ import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedactionIntegrationTest extends RulesIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules.drl"); @@ -117,19 +121,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); @@ -501,8 +496,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); - deleted.add("David Chubb"); - deleted.add("mouse"); + deletedInTemplate.add("David Chubb"); + deletedInTemplate.add("mouse"); reanlysisVersions.put("mouse", 3L); @@ -537,7 +532,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - deleted.remove("mouse"); + deletedInTemplate.remove("mouse"); reanlysisVersions.put("mouse", 4L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); @@ -657,8 +652,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); - deleted.add("David Chubb"); - deleted.add("mouse"); + deletedInTemplate.add("David Chubb"); + deletedInTemplate.add("mouse"); reanlysisVersions.put("David Chubb", 3L); @@ -695,7 +690,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - deleted.remove("mouse"); + deletedInTemplate.remove("mouse"); reanlysisVersions.put("mouse", 4L); mockDictionaryCalls(3L); @@ -1321,7 +1316,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .findFirst() .get(); - request.setManualRedactions(ManualRedactions.builder() .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() .annotationId("3029651d0842a625f2d23f8375c23600") @@ -1844,6 +1838,89 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { } + @Test + @SneakyThrows + @Order(1) + public void testShrinkTemplateEntryAndAddToDossierLevel() { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + String pdfFile = "files/syngenta/CustomerFiles/SinglePages/test1S1T1.pdf"; + + String manualAddId = UUID.randomUUID().toString(); + + List fullPositions = List.of(Rectangle.builder().topLeftX(190.85f).topLeftY(332.5033f).width(228.0134f).height(13.645125f).page(1).build()); + + List shrunkPositions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(333.45f).width(71.40744f).height(12.823441f).page(1).build()); + ManualRedactionEntry manualRedactionEntry = getFullManualRedactionEntry(manualAddId, fullPositions); + manualRedactionEntry.setAddToDictionary(true); + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + Set entriesToAdd = Set.of(manualRedactionEntry); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).build()); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + request.setAnalysisNumber(1); + + String commentsOnTheAssessmentReport = "Comments on the assessment report"; + dictionary.get(DICTIONARY_AUTHOR).add(commentsOnTheAssessmentReport); + reanlysisVersions.put(commentsOnTheAssessmentReport, 0L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + mockDictionaryCalls(0L); + + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertTrue(entityLog.getEntityLogEntry() + .stream() + .findFirst() + .isPresent()); + EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getValue().equals(commentsOnTheAssessmentReport)) + .findFirst() + .get(); + String id = entityLogEntry1.getId(); + + assertEquals(entityLogEntry1.getState(), EntryState.APPLIED); + + manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); + Set resizeRedactions = Set.of(getManualResizeRedaction(id, shrunkPositions)); + + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).resizeRedactions(resizeRedactions).build()); + request.setAnalysisNumber(2); + + dossierDictionary.get(DICTIONARY_AUTHOR).add(commentsOnTheAssessmentReport); + deletedInDossier.add(commentsOnTheAssessmentReport); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); + String assessment = "assessment"; + dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment); + reanlysisVersions.put(assessment, 1L); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(id))); + entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(id)) + .findFirst() + .get(); + var entityLogEntry2 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getValue().equals(assessment)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.IGNORED); + assertEquals(entityLogEntry2.getState(), EntryState.APPLIED); + } + + private IdRemoval getIdRemoval(String id) { return IdRemoval.builder() @@ -1859,18 +1936,52 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { private ManualRedactionEntry getManualRedactionEntry(String id, List positions, String reason) { - ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry(); - manualRedactionEntry2.setAnnotationId(id); - manualRedactionEntry2.setFileId("fileId"); - manualRedactionEntry2.setUser("test"); - manualRedactionEntry2.setType("manual"); - manualRedactionEntry2.setRectangle(false); - manualRedactionEntry2.setRequestDate(OffsetDateTime.now()); - manualRedactionEntry2.setValue("assessment"); - manualRedactionEntry2.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); - manualRedactionEntry2.setReason(reason); - manualRedactionEntry2.setPositions(positions); - return manualRedactionEntry2; + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(id); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("manual"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue("assessment"); + manualRedactionEntry.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + manualRedactionEntry.setReason(reason); + manualRedactionEntry.setPositions(positions); + return manualRedactionEntry; + } + + + private ManualRedactionEntry getFullManualRedactionEntry(String id, List positions) { + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(id); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("manual"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue("Comments on the assessment report"); + manualRedactionEntry.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + manualRedactionEntry.setReason( + "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); + manualRedactionEntry.setPositions(positions); + return manualRedactionEntry; + } + + + private ManualResizeRedaction getManualResizeRedaction(String id, List positions) { + + ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction(); + manualResizeRedaction.setAnnotationId(id); + manualResizeRedaction.setFileId("fileId"); + manualResizeRedaction.setUser("test"); + manualResizeRedaction.setRequestDate(OffsetDateTime.now()); + manualResizeRedaction.setValue("assessment"); + manualResizeRedaction.setValue("assessment"); + manualResizeRedaction.setPositions(positions); + manualResizeRedaction.setAddToAllDossiers(false); + manualResizeRedaction.setUpdateDictionary(true); + return manualResizeRedaction; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 0b34df29..78f59de0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -74,19 +74,10 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 0f4352b4..e23dc2ae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -269,19 +269,10 @@ public class RulesTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); mockDictionaryCalls(0L); @@ -774,21 +765,40 @@ public class RulesTest { } - private List getTypeResponse() { + protected List getTemplateDictionaryTypeResponse() { - return typeColorMap.entrySet() + return dictionary.keySet() .stream() - .map(typeColor -> Type.builder() - .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(typeColor.getKey()) + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(key) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor(typeColor.getValue()) - .isHint(hintTypeMap.get(typeColor.getKey())) - .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) - .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) - .rank(rankTypeMap.get(typeColor.getKey())) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) .build()) + .collect(Collectors.toList()); + } + + + protected List getDossierDictionaryTypeResponse() { + + return dossierDictionary.keySet() + .stream() + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID) + .type(key) + .dossierId(TEST_DOSSIER_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) + .build()) .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 8692f648..75946071 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -108,19 +108,10 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index e5bc9af8..140f8811 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -122,20 +122,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTemplateDictionaryTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index ee11e966..fc58dce0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -98,20 +98,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); - + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);