From 2a56187294c125931d4b998ee1f812ea426e2b58 Mon Sep 17 00:00:00 2001 From: Thomas Beyer Date: Wed, 29 Mar 2023 09:35:55 +0200 Subject: [PATCH] RED-6411 - optimize code --- .../AbstractRedactionIntegrationTest.java | 379 +++++++++--------- .../v1/server/RedactionIntegrationTest.java | 121 +++--- ...t.java => RedactionIntegrationTestV2.java} | 31 +- 3 files changed, 254 insertions(+), 277 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/{RedactionIntegrationWithRulesV2Test.java => RedactionIntegrationTestV2.java} (85%) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 70202114..4cdf09c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -2,13 +2,10 @@ package com.iqser.red.service.redaction.v1.server; import static org.mockito.Mockito.when; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URL; -import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; @@ -50,26 +47,48 @@ import lombok.SneakyThrows; public abstract class AbstractRedactionIntegrationTest { - protected static final String VERTEBRATE = "vertebrate"; - protected static final String ADDRESS = "CBI_address"; - protected static final String AUTHOR = "CBI_author"; - protected static final String SPONSOR = "CBI_sponsor"; + protected static final String VERTEBRATE_INDICATOR = "vertebrate"; + protected static final String DICTIONARY_ADDRESS = "CBI_address"; + protected static final String DICTIONARY_AUTHOR = "CBI_author"; + protected static final String DICTIONARY_SPONSOR = "CBI_sponsor"; + protected static final String DICTIONARY_PII = "PII"; protected static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; protected static final String REDACTION_INDICATOR = "redaction_indicator"; - protected static final String HINT_ONLY = "hint_only"; - protected static final String MUST_REDACT = "must_redact"; - protected static final String PUBLISHED_INFORMATION = "published_information"; - protected static final String TEST_METHOD = "test_method"; - protected static final String PURITY = "purity"; - protected static final String IMAGE = "image"; - protected static final String LOGO = "logo"; - protected static final String SIGNATURE = "signature"; - protected static final String FORMULA = "formula"; - protected static final String OCR = "ocr"; - protected static final String DOSSIER_REDACTIONS = "dossier_redactions"; - protected static final String IMPORTED_REDACTION = "imported_redaction"; - protected static final String PII = "PII"; - protected static final String ROTATE_SIMPLE = "RotateSimple"; + protected static final String HINT_ONLY_INDICATOR = "hint_only"; + protected static final String MUST_REDACT_INDICATOR = "must_redact"; + protected static final String PUBLISHED_INFORMATION_INDICATOR = "published_information"; + protected static final String TEST_METHOD_INDICATOR = "test_method"; + protected static final String PURITY_INDICATOR = "purity"; + protected static final String IMAGE_INDICATOR = "image"; + protected static final String LOGO_INDICATOR = "logo"; + protected static final String SIGNATURE_INDICATOR = "signature"; + protected static final String FORMULA_INDICATOR = "formula"; + protected static final String OCR_INDICATOR = "ocr"; + protected static final String DOSSIER_REDACTIONS_INDICATOR = "dossier_redactions"; + protected static final String IMPORTED_REDACTION_INDICATOR = "imported_redaction"; + protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple"; + + protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; + public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String FORMULA_TYPE_ID = FORMULA_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String SIGNATURE_TYPE_ID = SIGNATURE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String LOGO_TYPE_ID = LOGO_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String OCR_TYPE_ID = OCR_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String IMAGE_TYPE_ID = IMAGE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String PURITY_TYPE_ID = PURITY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @Autowired protected RedactionController redactionController; @@ -114,7 +133,6 @@ public abstract class AbstractRedactionIntegrationTest { protected final Map reanlysisVersions = new HashMap<>(); protected final Set deleted = new HashSet<>(); - protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; protected final static String TEST_DOSSIER_ID = "123"; protected final static String TEST_FILE_ID = "123"; @@ -136,85 +154,71 @@ public abstract class AbstractRedactionIntegrationTest { protected void mockDictionaryCalls(Long version) { - when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, - false)); - when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ADDRESS, false)); - when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SPONSOR, false)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( - NO_REDACTION_INDICATOR, - false)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( - REDACTION_INDICATOR, - false)); - when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT, - false)); - when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( - PUBLISHED_INFORMATION, - false)); - when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD, - false)); - when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PII, false)); - when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY, false)); - when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE, false)); - when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(OCR, false)); - when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO, false)); - when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE, false)); - when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA, false)); - when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE, - false)); - when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( - DOSSIER_REDACTIONS, - true)); - when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( - IMPORTED_REDACTION, - true)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(ADDRESS_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_PII, false)); + when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(IMAGE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(OCR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(OCR_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(LOGO_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,true)); + when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,true)); } protected void loadDictionaryForTest() { - dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) + dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) + dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) + dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) + dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) + dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) + dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) + dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) + dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) + dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) + dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) + dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) + dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) + dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) + dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) + dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) + dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); - dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); + dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); - falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) + falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); } @@ -223,153 +227,128 @@ public abstract class AbstractRedactionIntegrationTest { protected void loadOnlyDictionaryForSimpleFile() { dictionary.clear(); - dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>()) + dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); } + @SneakyThrows protected static String loadFromClassPath(String path) { URL resource = ResourceLoader.class.getClassLoader().getResource(path); if (resource == null) { throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); } - try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String str; - while ((str = br.readLine()) != null) { - sb.append(str).append("\n"); - } - return sb.toString(); - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + path, e); - } - } - - - protected List getPathsRecursively(File path) { - - List result = new ArrayList<>(); - if (path == null || path.listFiles() == null) { - return result; - } - for (File f : path.listFiles()) { - if (f.isFile()) { - result.add(f); - } else { - result.addAll(getPathsRecursively(f)); - } - } - return result; - + List stringList = Files.readAllLines(new File(resource.getPath()).toPath()); + return String.join("\n", stringList); } protected void loadTypeForTest() { - typeColorMap.put(VERTEBRATE, "#ff85f7"); - typeColorMap.put(ADDRESS, "#ffe187"); - typeColorMap.put(AUTHOR, "#ffe187"); - typeColorMap.put(SPONSOR, "#85ebff"); + typeColorMap.put(VERTEBRATE_INDICATOR, "#ff85f7"); + typeColorMap.put(DICTIONARY_ADDRESS, "#ffe187"); + typeColorMap.put(DICTIONARY_AUTHOR, "#ffe187"); + typeColorMap.put(DICTIONARY_SPONSOR, "#85ebff"); typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff"); typeColorMap.put(REDACTION_INDICATOR, "#caff85"); - typeColorMap.put(HINT_ONLY, "#abc0c4"); - typeColorMap.put(MUST_REDACT, "#fab4c0"); - typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff"); - typeColorMap.put(TEST_METHOD, "#91fae8"); - typeColorMap.put(PII, "#66ccff"); - typeColorMap.put(PURITY, "#ffe187"); - typeColorMap.put(IMAGE, "#fcc5fb"); - typeColorMap.put(OCR, "#fcc5fb"); - typeColorMap.put(LOGO, "#ffe187"); - typeColorMap.put(FORMULA, "#ffe187"); - typeColorMap.put(SIGNATURE, "#ffe187"); - typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6"); - typeColorMap.put(ROTATE_SIMPLE, "#66ccff"); + typeColorMap.put(HINT_ONLY_INDICATOR, "#abc0c4"); + typeColorMap.put(MUST_REDACT_INDICATOR, "#fab4c0"); + typeColorMap.put(PUBLISHED_INFORMATION_INDICATOR, "#85ebff"); + typeColorMap.put(TEST_METHOD_INDICATOR, "#91fae8"); + typeColorMap.put(DICTIONARY_PII, "#66ccff"); + typeColorMap.put(PURITY_INDICATOR, "#ffe187"); + typeColorMap.put(IMAGE_INDICATOR, "#fcc5fb"); + typeColorMap.put(OCR_INDICATOR, "#fcc5fb"); + typeColorMap.put(LOGO_INDICATOR, "#ffe187"); + typeColorMap.put(FORMULA_INDICATOR, "#ffe187"); + typeColorMap.put(SIGNATURE_INDICATOR, "#ffe187"); + typeColorMap.put(IMPORTED_REDACTION_INDICATOR, "#fcfbe6"); + typeColorMap.put(ROTATE_SIMPLE_INDICATOR, "#66ccff"); - hintTypeMap.put(VERTEBRATE, true); - hintTypeMap.put(ADDRESS, false); - hintTypeMap.put(AUTHOR, false); - hintTypeMap.put(SPONSOR, false); + hintTypeMap.put(VERTEBRATE_INDICATOR, true); + hintTypeMap.put(DICTIONARY_ADDRESS, false); + hintTypeMap.put(DICTIONARY_AUTHOR, false); + hintTypeMap.put(DICTIONARY_SPONSOR, false); hintTypeMap.put(NO_REDACTION_INDICATOR, true); hintTypeMap.put(REDACTION_INDICATOR, true); - hintTypeMap.put(HINT_ONLY, true); - hintTypeMap.put(MUST_REDACT, true); - hintTypeMap.put(PUBLISHED_INFORMATION, true); - hintTypeMap.put(TEST_METHOD, true); - hintTypeMap.put(PII, false); - hintTypeMap.put(PURITY, false); - hintTypeMap.put(IMAGE, true); - hintTypeMap.put(OCR, true); - hintTypeMap.put(FORMULA, false); - hintTypeMap.put(LOGO, false); - hintTypeMap.put(SIGNATURE, false); - hintTypeMap.put(DOSSIER_REDACTIONS, false); - hintTypeMap.put(IMPORTED_REDACTION, false); - hintTypeMap.put(ROTATE_SIMPLE, false); + hintTypeMap.put(HINT_ONLY_INDICATOR, true); + hintTypeMap.put(MUST_REDACT_INDICATOR, true); + hintTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, true); + hintTypeMap.put(TEST_METHOD_INDICATOR, true); + hintTypeMap.put(DICTIONARY_PII, false); + hintTypeMap.put(PURITY_INDICATOR, false); + hintTypeMap.put(IMAGE_INDICATOR, true); + hintTypeMap.put(OCR_INDICATOR, true); + hintTypeMap.put(FORMULA_INDICATOR, false); + hintTypeMap.put(LOGO_INDICATOR, false); + hintTypeMap.put(SIGNATURE_INDICATOR, false); + hintTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false); + hintTypeMap.put(IMPORTED_REDACTION_INDICATOR, false); + hintTypeMap.put(ROTATE_SIMPLE_INDICATOR, false); - caseInSensitiveMap.put(VERTEBRATE, true); - caseInSensitiveMap.put(ADDRESS, false); - caseInSensitiveMap.put(AUTHOR, false); - caseInSensitiveMap.put(SPONSOR, false); + caseInSensitiveMap.put(VERTEBRATE_INDICATOR, true); + caseInSensitiveMap.put(DICTIONARY_ADDRESS, false); + caseInSensitiveMap.put(DICTIONARY_AUTHOR, false); + caseInSensitiveMap.put(DICTIONARY_SPONSOR, false); caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); caseInSensitiveMap.put(REDACTION_INDICATOR, true); - caseInSensitiveMap.put(HINT_ONLY, true); - caseInSensitiveMap.put(MUST_REDACT, true); - caseInSensitiveMap.put(PUBLISHED_INFORMATION, true); - caseInSensitiveMap.put(TEST_METHOD, false); - caseInSensitiveMap.put(PII, false); - caseInSensitiveMap.put(PURITY, false); - caseInSensitiveMap.put(IMAGE, true); - caseInSensitiveMap.put(OCR, true); - caseInSensitiveMap.put(SIGNATURE, true); - caseInSensitiveMap.put(LOGO, true); - caseInSensitiveMap.put(FORMULA, true); - caseInSensitiveMap.put(DOSSIER_REDACTIONS, false); - caseInSensitiveMap.put(IMPORTED_REDACTION, false); - caseInSensitiveMap.put(ROTATE_SIMPLE, true); + caseInSensitiveMap.put(HINT_ONLY_INDICATOR, true); + caseInSensitiveMap.put(MUST_REDACT_INDICATOR, true); + caseInSensitiveMap.put(PUBLISHED_INFORMATION_INDICATOR, true); + caseInSensitiveMap.put(TEST_METHOD_INDICATOR, false); + caseInSensitiveMap.put(DICTIONARY_PII, false); + caseInSensitiveMap.put(PURITY_INDICATOR, false); + caseInSensitiveMap.put(IMAGE_INDICATOR, true); + caseInSensitiveMap.put(OCR_INDICATOR, true); + caseInSensitiveMap.put(SIGNATURE_INDICATOR, true); + caseInSensitiveMap.put(LOGO_INDICATOR, true); + caseInSensitiveMap.put(FORMULA_INDICATOR, true); + caseInSensitiveMap.put(DOSSIER_REDACTIONS_INDICATOR, false); + caseInSensitiveMap.put(IMPORTED_REDACTION_INDICATOR, false); + caseInSensitiveMap.put(ROTATE_SIMPLE_INDICATOR, true); - recommendationTypeMap.put(VERTEBRATE, false); - recommendationTypeMap.put(ADDRESS, false); - recommendationTypeMap.put(AUTHOR, false); - recommendationTypeMap.put(SPONSOR, false); + recommendationTypeMap.put(VERTEBRATE_INDICATOR, false); + recommendationTypeMap.put(DICTIONARY_ADDRESS, false); + recommendationTypeMap.put(DICTIONARY_AUTHOR, false); + recommendationTypeMap.put(DICTIONARY_SPONSOR, false); recommendationTypeMap.put(NO_REDACTION_INDICATOR, false); recommendationTypeMap.put(REDACTION_INDICATOR, false); - recommendationTypeMap.put(HINT_ONLY, false); - recommendationTypeMap.put(MUST_REDACT, false); - recommendationTypeMap.put(PUBLISHED_INFORMATION, false); - recommendationTypeMap.put(TEST_METHOD, false); - recommendationTypeMap.put(PII, false); - recommendationTypeMap.put(PURITY, false); - recommendationTypeMap.put(IMAGE, false); - recommendationTypeMap.put(OCR, false); - recommendationTypeMap.put(FORMULA, false); - recommendationTypeMap.put(SIGNATURE, false); - recommendationTypeMap.put(LOGO, false); - recommendationTypeMap.put(DOSSIER_REDACTIONS, false); - recommendationTypeMap.put(IMPORTED_REDACTION, false); - recommendationTypeMap.put(ROTATE_SIMPLE, false); + recommendationTypeMap.put(HINT_ONLY_INDICATOR, false); + recommendationTypeMap.put(MUST_REDACT_INDICATOR, false); + recommendationTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, false); + recommendationTypeMap.put(TEST_METHOD_INDICATOR, false); + recommendationTypeMap.put(DICTIONARY_PII, false); + recommendationTypeMap.put(PURITY_INDICATOR, false); + recommendationTypeMap.put(IMAGE_INDICATOR, false); + recommendationTypeMap.put(OCR_INDICATOR, false); + recommendationTypeMap.put(FORMULA_INDICATOR, false); + recommendationTypeMap.put(SIGNATURE_INDICATOR, false); + recommendationTypeMap.put(LOGO_INDICATOR, false); + recommendationTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false); + recommendationTypeMap.put(IMPORTED_REDACTION_INDICATOR, false); + recommendationTypeMap.put(ROTATE_SIMPLE_INDICATOR, false); - rankTypeMap.put(PURITY, 155); - rankTypeMap.put(PII, 150); - rankTypeMap.put(ADDRESS, 140); - rankTypeMap.put(AUTHOR, 130); - rankTypeMap.put(SPONSOR, 120); - rankTypeMap.put(VERTEBRATE, 110); - rankTypeMap.put(MUST_REDACT, 100); + rankTypeMap.put(PURITY_INDICATOR, 155); + rankTypeMap.put(DICTIONARY_PII, 150); + rankTypeMap.put(DICTIONARY_ADDRESS, 140); + rankTypeMap.put(DICTIONARY_AUTHOR, 130); + rankTypeMap.put(DICTIONARY_SPONSOR, 120); + rankTypeMap.put(VERTEBRATE_INDICATOR, 110); + rankTypeMap.put(MUST_REDACT_INDICATOR, 100); rankTypeMap.put(REDACTION_INDICATOR, 90); rankTypeMap.put(NO_REDACTION_INDICATOR, 80); - rankTypeMap.put(PUBLISHED_INFORMATION, 70); - rankTypeMap.put(TEST_METHOD, 60); - rankTypeMap.put(HINT_ONLY, 50); - rankTypeMap.put(IMAGE, 30); - rankTypeMap.put(OCR, 29); - rankTypeMap.put(LOGO, 28); - rankTypeMap.put(SIGNATURE, 27); - rankTypeMap.put(FORMULA, 26); - rankTypeMap.put(DOSSIER_REDACTIONS, 200); - rankTypeMap.put(IMPORTED_REDACTION, 200); - rankTypeMap.put(ROTATE_SIMPLE, 150); + rankTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, 70); + rankTypeMap.put(TEST_METHOD_INDICATOR, 60); + rankTypeMap.put(HINT_ONLY_INDICATOR, 50); + rankTypeMap.put(IMAGE_INDICATOR, 30); + rankTypeMap.put(OCR_INDICATOR, 29); + rankTypeMap.put(LOGO_INDICATOR, 28); + rankTypeMap.put(SIGNATURE_INDICATOR, 27); + rankTypeMap.put(FORMULA_INDICATOR, 26); + rankTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, 200); + rankTypeMap.put(IMPORTED_REDACTION_INDICATOR, 200); + rankTypeMap.put(ROTATE_SIMPLE_INDICATOR, 150); colors.setSkippedColor("#cccccc"); colors.setRequestAddColor("#04b093"); @@ -433,17 +412,19 @@ public abstract class AbstractRedactionIntegrationTest { if (entries == null) { entries = Collections.emptyList(); } + return entries.stream().map(this::toDictionaryEntry).collect(Collectors.toList()); - List dictionaryEntries = new ArrayList<>(); - entries.forEach(entry -> { - dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build()); - }); - return dictionaryEntries; + } + + + private DictionaryEntry toDictionaryEntry(String entry) { + + return DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build(); } @SneakyThrows - protected AnalyzeRequest prepareStorage(String file) { + protected AnalyzeRequest uploadFileToStorage(String file) { return prepareStorage(file, "files/cv_service_empty_response.json"); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 599179c9..b5b33dca 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -4,12 +4,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; @@ -79,7 +82,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) - public static class RedactionIntegrationTestConfiguration { + static class RedactionIntegrationTestConfiguration { @Bean public KieContainer kieContainer() { @@ -123,18 +126,17 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS) + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) .dossierTemplateId(TEST_DOSSIER_ID) .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .build())); mockDictionaryCalls(null); - mockDictionaryCalls(0L); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -143,7 +145,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void test270Rotated() { - AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/270Rotated.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); assertThat(result).isNotNull(); @@ -154,7 +156,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Disabled public void testLargeScannedFileOOM() { - AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); + AnalyzeRequest request = uploadFileToStorage("scanned/VV-377031.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); assertThat(result).isNotNull(); @@ -166,7 +168,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/Minimal Examples/merge_images.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/merge_images.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -180,7 +182,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1)); - dictionary.get(AUTHOR).add("Drinking water"); + dictionary.get(DICTIONARY_AUTHOR).add("Drinking water"); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); @@ -206,7 +208,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { // F. Lastname, J. Doe, M. Mustermann // Lastname M., Doe J., Mustermann M. - AnalyzeRequest request = prepareStorage("files/Minimal Examples/ExpansionTest.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/ExpansionTest.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -222,7 +224,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -245,7 +247,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void testAddFileAttribute() { - AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf"); + AnalyzeRequest request = uploadFileToStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -262,7 +264,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("testIgnoreHint"); - AnalyzeRequest request = prepareStorage("files/new/test-ignore-hint.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/test-ignore-hint.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); @@ -300,15 +302,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("noExceptionShouldBeThrownForAnyFiles"); ClassLoader loader = getClass().getClassLoader(); URL url = loader.getResource("files"); - File[] files = new File(url.getPath()).listFiles(); - List input = new ArrayList<>(); - for (File file : files) { - input.addAll(getPathsRecursively(file)); - } - for (File path : input) { + Path path = Paths.get(URI.create(url.toString())); - AnalyzeRequest request = prepareStorage(path.getPath()); - System.out.println("Redacting file : " + path.getName()); + Files.walk(path).forEach(currentPath -> { + AnalyzeRequest request = uploadFileToStorage(currentPath.toString()); + System.out.println("Redacting file : " + currentPath.getFileName()); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); long fstart = System.currentTimeMillis(); @@ -327,7 +325,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { assertThat(entry.getValue().size()).isEqualTo(1); }); - dictionary.get(AUTHOR).add("Drinking water"); + dictionary.get(DICTIONARY_AUTHOR).add("Drinking water"); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); long rstart = System.currentTimeMillis(); @@ -335,8 +333,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { long rend = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (rend - rstart)); - - } + }); long end = System.currentTimeMillis(); @@ -352,7 +349,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf"; long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage(fileName); + AnalyzeRequest request = uploadFileToStorage(fileName); request.setExcludedPages(Set.of(1)); request.setFileAttributes(List.of(FileAttribute.builder() @@ -396,10 +393,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); - dictionary.get(AUTHOR).add("properties"); + dictionary.get(DICTIONARY_AUTHOR).add("properties"); reanlysisVersions.put("properties", 1L); - dictionary.get(AUTHOR).add("physical"); + dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); deleted.add("David Chubb"); @@ -409,7 +406,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false)); start = System.currentTimeMillis(); @@ -442,7 +439,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false)); analyzeService.reanalyze(request); @@ -458,19 +455,19 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String fileName = "files/new/test1S1T1.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; - AnalyzeRequest request = prepareStorage(fileName); + AnalyzeRequest request = uploadFileToStorage(fileName); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); - dictionary.get(AUTHOR).add("report"); + dictionary.get(DICTIONARY_AUTHOR).add("report"); reanlysisVersions.put("report", 2L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L); mockDictionaryCalls(0L); analyzeService.reanalyze(request); - dictionary.get(AUTHOR).add("assessment report"); + dictionary.get(DICTIONARY_AUTHOR).add("assessment report"); reanlysisVersions.put("assessment report", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); mockDictionaryCalls(2L); @@ -506,7 +503,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { responseJson.getInputStream()); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage(fileName); + AnalyzeRequest request = uploadFileToStorage(fileName); request.setExcludedPages(Set.of(1)); request.setFileAttributes(List.of(FileAttribute.builder() @@ -550,11 +547,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); - dictionary.get(AUTHOR).add("properties"); + dictionary.get(DICTIONARY_AUTHOR).add("properties"); reanlysisVersions.put("properties", 1L); mockDictionaryCalls(0L); - dictionary.get(AUTHOR).add("physical"); + dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); deleted.add("David Chubb"); @@ -566,7 +563,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false)); start = System.currentTimeMillis(); @@ -600,7 +597,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); - when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false)); analyzeService.reanalyze(request); @@ -623,7 +620,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { }; List types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); - AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/PublishedInformationTest.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); ManualRedactions manualRedactions = new ManualRedactions(); manualRedactions.getIdsToRemove() @@ -681,7 +678,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("testTableRedaction"); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -750,7 +747,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/unicodeProblem.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -772,7 +769,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("testTableRedaction"); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFile.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -798,7 +795,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("testTableRedaction"); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/new/RotateTestFileSimple.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFileSimple.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -820,7 +817,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("testTableHeader"); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/Minimal Examples/NoHeaderTable.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/NoHeaderTable.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -855,7 +852,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/new/S157.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf"); ManualRedactions manualRedactions = new ManualRedactions(); @@ -931,7 +928,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); - AnalyzeRequest request = prepareStorage(pdfFile); + AnalyzeRequest request = uploadFileToStorage(pdfFile); request.setManualRedactions(manualRedactions); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -964,7 +961,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void phantomCellsDocumentTest() throws IOException { - AnalyzeRequest request = prepareStorage("files/Minimal Examples/Phantom Cells.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/Phantom Cells.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -984,7 +981,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/Minimal Examples/sponsor_companies.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/sponsor_companies.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1046,7 +1043,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { // manualRedactions.getEntriesToAdd().add(manualRedactionEntry); - AnalyzeRequest request = prepareStorage(pdfFile); + AnalyzeRequest request = uploadFileToStorage(pdfFile); request.setManualRedactions(manualRedactions); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1102,7 +1099,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("expandByRegex"); long start = System.currentTimeMillis(); - AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1157,7 +1154,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); - AnalyzeRequest request = prepareStorage(pdfFile); + AnalyzeRequest request = uploadFileToStorage(pdfFile); request.setManualRedactions(manualRedactions); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); @@ -1187,7 +1184,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json"); - AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream()); @@ -1220,17 +1217,17 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void testExpandByPrefixRegEx() throws IOException { - assertThat(dictionary.get(AUTHOR)).contains("Robinson"); - assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson"); - assertThat(dictionary.get(AUTHOR)).contains("Bojangles"); - assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles"); - assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man"); - assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Robinson"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mrs. Robinson"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Bojangles"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Bojangles"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Tambourine Man"); + assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Tambourine Man"); String fileName = "files/mr-mrs.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; - AnalyzeRequest request = prepareStorage(fileName); + AnalyzeRequest request = uploadFileToStorage(fileName); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationWithRulesV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTestV2.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationWithRulesV2Test.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTestV2.java index 7b3d6919..b8f4d592 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationWithRulesV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTestV2.java @@ -6,7 +6,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -40,14 +40,14 @@ import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RedactionIntegrationWithRulesV2Test.RedactionIntegrationTestConfiguration.class) -public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegrationTest { +@Import(RedactionIntegrationTestV2.RedactionIntegrationTestConfiguration.class) +public class RedactionIntegrationTestV2 extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) - public static class RedactionIntegrationTestConfiguration { + static class RedactionIntegrationTestConfiguration { @Bean public KieContainer kieContainer() { @@ -91,18 +91,17 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS) + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) .dossierTemplateId(TEST_DOSSIER_ID) .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .build())); mockDictionaryCalls(null); - mockDictionaryCalls(0L); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -118,14 +117,14 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr @SneakyThrows public void testTermIsInTwoDictionariesAndInOneFalsePositive() { - AnalyzeRequest request = prepareStorage("files/new/simplified2.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf"); dictionary.clear(); - dictionary.computeIfAbsent(PII, v -> new ArrayList<>()).add("Dr. Alan Miller"); - dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()).add("Dr. Alan Miller"); + dictionary.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller")); + dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Dr. Alan Miller")); falsePositive.clear(); - falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()).add("Dr. Alan Miller COMPLETION DATE:"); + falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:")); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); @@ -136,7 +135,7 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); - assertThat(redactionLogEntry.getType()).isEqualTo("CBI_author"); + assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getValue()).isEqualTo("Dr. Alan Miller"); assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);