From 18966be46074c7fc1920dd544359c46218b9ded0 Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Thu, 13 Jun 2024 08:36:57 +0200 Subject: [PATCH] RED-9187 - Annotation does not switch back to redacted if published... --- .../v1/server/service/DictionaryService.java | 6 ++ .../AbstractRedactionIntegrationTest.java | 8 +- .../v1/server/RedactionAcceptanceTest.java | 98 ++++++++++++++++++- .../v1/server/RedactionIntegrationTest.java | 2 +- 4 files changed, 107 insertions(+), 7 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index 325f30e1..895f535b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -74,6 +74,12 @@ public class DictionaryService { } + public void clearTenantDictionaryCache() { + + tenantDictionaryCache.invalidateAll(); + } + + @SneakyThrows @Observed(name = "DictionaryService", contextualName = "update-dictionary") @Timed("redactmanager_updateDictionary") diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index e116014b..4e1f484b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -100,6 +100,8 @@ public abstract class AbstractRedactionIntegrationTest { protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple"; protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; + protected final static String TEST_DOSSIER_ID = "123"; + protected final static String TEST_FILE_ID = "123"; public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -112,6 +114,7 @@ public abstract class AbstractRedactionIntegrationTest { public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID+ ":" + TEST_DOSSIER_ID; public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -181,8 +184,6 @@ public abstract class AbstractRedactionIntegrationTest { protected final Map reanlysisVersions = new HashMap<>(); protected final Set deleted = new HashSet<>(); - protected final static String TEST_DOSSIER_ID = "123"; - protected final static String TEST_FILE_ID = "123"; @MockBean protected RulesClient rulesClient; @@ -249,6 +250,8 @@ public abstract class AbstractRedactionIntegrationTest { true)); when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR, true)); + when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, + true)); } @@ -346,6 +349,7 @@ public abstract class AbstractRedactionIntegrationTest { .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); + dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index fb314ba9..66c71571 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.when; import java.io.FileOutputStream; import java.io.IOException; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Stream; @@ -14,6 +15,7 @@ import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -40,6 +42,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; @@ -54,6 +57,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); + @Autowired + DictionaryService dictionaryService; + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @Import(LayoutParsingServiceProcessorConfiguration.class) @@ -87,14 +93,26 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .id(DOSSIER_REDACTIONS_TYPE_ID) .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) .hexColor("#ffe187") .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build(), + Type.builder() + .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID) + .type(PUBLISHED_INFORMATION_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .hexColor("#ffe180") + .isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) + .rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) .build())); mockDictionaryCalls(null); @@ -128,6 +146,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); + dictionaryService.clearTenantDictionaryCache(); AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); @@ -141,7 +161,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() .orElseThrow(); - assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + assertThat(asyaLyon1.getSection().startsWith("Paragraph:")); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); @@ -175,7 +195,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); - AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); System.out.println("Finished structure analysis"); @@ -220,7 +240,77 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); } + dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove("Press"); + assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse(); + } + @Test + public void testPublishedInformationRemovalAtDossierLevel() throws IOException { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf"); + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + System.out.println("Finished structure analysis"); + mockDictionaryCalls(0L); + AnalyzeResult result = analyzeService.analyze(request); + System.out.println("Finished analysis"); + // add dossier template published information and the cbi_author + var publishedInfoValue = "great job"; + var cbiAuthorValue = "Kara Hunt"; + dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue); + dictionary.get(DICTIONARY_AUTHOR).add(cbiAuthorValue); + reanlysisVersions.put(publishedInfoValue, 3L); + reanlysisVersions.put(cbiAuthorValue, 3L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); + mockDictionaryCalls(2L); + + request.setAnalysisNumber(3); + analyzeService.reanalyze(request); + + EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst() + .orElseThrow(); + var cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst() + .orElseThrow(); + assertEquals(EntryState.SKIPPED, cbiAuthor.getState()); + + // remove at dossier level + var idRemoval = buildIdRemoval(publishedInformationEntry.getId()); + idRemoval.setRemoveFromDictionary(true); + + var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build(); + request.setManualRedactions(manualRedactions); + + deleted.add(publishedInfoValue); + reanlysisVersions.put(publishedInfoValue, 4L); + + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(3L); + dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue); + + request.setAnalysisNumber(3); + mockDictionaryCalls(3L); + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst() + .orElseThrow(); + cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst() + .orElseThrow(); + + assertEquals(EntryState.APPLIED, cbiAuthor.getState()); + + // cleanup changes + deleted.remove(publishedInfoValue); + assertThat(deleted.contains(publishedInfoValue)).isFalse(); + dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); + assertThat(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).size()).isEqualTo(0); + dictionary.get(DICTIONARY_AUTHOR).remove(cbiAuthorValue); + assertThat(dictionary.get(DICTIONARY_AUTHOR).contains(cbiAuthorValue)).isFalse(); + dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove(publishedInfoValue); + assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains(publishedInfoValue)).isFalse(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 6306430e..1593a25c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -119,7 +119,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .type(DOSSIER_REDACTIONS_INDICATOR)