Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6411 - optimize code

Browse Source
This commit is contained in:
Thomas Beyer 2023-03-29 09:35:55 +02:00
parent 710747d00b
commit 2a56187294
3 changed files with 254 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

379
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -2,13 +2,10 @@ package com.iqser.red.service.redaction.v1.server;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.file.Files;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -50,26 +47,48 @@ import lombok.SneakyThrows;
public abstract class AbstractRedactionIntegrationTest { public abstract class AbstractRedactionIntegrationTest {
protected static final String VERTEBRATE = "vertebrate"; protected static final String VERTEBRATE_INDICATOR = "vertebrate";
protected static final String ADDRESS = "CBI_address"; protected static final String DICTIONARY_ADDRESS = "CBI_address";
protected static final String AUTHOR = "CBI_author"; protected static final String DICTIONARY_AUTHOR = "CBI_author";
protected static final String SPONSOR = "CBI_sponsor"; protected static final String DICTIONARY_SPONSOR = "CBI_sponsor";
protected static final String DICTIONARY_PII = "PII";
protected static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; protected static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
protected static final String REDACTION_INDICATOR = "redaction_indicator"; protected static final String REDACTION_INDICATOR = "redaction_indicator";
protected static final String HINT_ONLY = "hint_only"; protected static final String HINT_ONLY_INDICATOR = "hint_only";
protected static final String MUST_REDACT = "must_redact"; protected static final String MUST_REDACT_INDICATOR = "must_redact";
protected static final String PUBLISHED_INFORMATION = "published_information"; protected static final String PUBLISHED_INFORMATION_INDICATOR = "published_information";
protected static final String TEST_METHOD = "test_method"; protected static final String TEST_METHOD_INDICATOR = "test_method";
protected static final String PURITY = "purity"; protected static final String PURITY_INDICATOR = "purity";
protected static final String IMAGE = "image"; protected static final String IMAGE_INDICATOR = "image";
protected static final String LOGO = "logo"; protected static final String LOGO_INDICATOR = "logo";
protected static final String SIGNATURE = "signature"; protected static final String SIGNATURE_INDICATOR = "signature";
protected static final String FORMULA = "formula"; protected static final String FORMULA_INDICATOR = "formula";
protected static final String OCR = "ocr"; protected static final String OCR_INDICATOR = "ocr";
protected static final String DOSSIER_REDACTIONS = "dossier_redactions"; protected static final String DOSSIER_REDACTIONS_INDICATOR = "dossier_redactions";
protected static final String IMPORTED_REDACTION = "imported_redaction"; protected static final String IMPORTED_REDACTION_INDICATOR = "imported_redaction";
protected static final String PII = "PII"; protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple";
protected static final String ROTATE_SIMPLE = "RotateSimple";
protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String FORMULA_TYPE_ID = FORMULA_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String SIGNATURE_TYPE_ID = SIGNATURE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String LOGO_TYPE_ID = LOGO_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String OCR_TYPE_ID = OCR_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String IMAGE_TYPE_ID = IMAGE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String PURITY_TYPE_ID = PURITY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
@Autowired @Autowired
protected RedactionController redactionController; protected RedactionController redactionController;
@ -114,7 +133,6 @@ public abstract class AbstractRedactionIntegrationTest {
protected final Map<String, Long> reanlysisVersions = new HashMap<>(); protected final Map<String, Long> reanlysisVersions = new HashMap<>();
protected final Set<String> deleted = new HashSet<>(); protected final Set<String> deleted = new HashSet<>();
protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
protected final static String TEST_DOSSIER_ID = "123"; protected final static String TEST_DOSSIER_ID = "123";
protected final static String TEST_FILE_ID = "123"; protected final static String TEST_FILE_ID = "123";
@ -136,85 +154,71 @@ public abstract class AbstractRedactionIntegrationTest {
protected void mockDictionaryCalls(Long version) { protected void mockDictionaryCalls(Long version) {
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE, when(dictionaryClient.getDictionaryForType(VERTEBRATE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(ADDRESS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false)); when(dictionaryClient.getDictionaryForType(AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false)); when(dictionaryClient.getDictionaryForType(SPONSOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_SPONSOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse( when(dictionaryClient.getDictionaryForType(REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
NO_REDACTION_INDICATOR, when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse( when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, false));
REDACTION_INDICATOR, when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_PII, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false)); when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT, when(dictionaryClient.getDictionaryForType(IMAGE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(OCR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse( when(dictionaryClient.getDictionaryForType(LOGO_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO_INDICATOR, false));
PUBLISHED_INFORMATION, when(dictionaryClient.getDictionaryForType(SIGNATURE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD, when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false));
false)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,true));
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,true));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
DOSSIER_REDACTIONS,
true));
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
IMPORTED_REDACTION,
true));
} }
protected void loadDictionaryForTest() { protected void loadDictionaryForTest() {
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
} }
@ -223,153 +227,128 @@ public abstract class AbstractRedactionIntegrationTest {
protected void loadOnlyDictionaryForSimpleFile() { protected void loadOnlyDictionaryForSimpleFile() {
dictionary.clear(); dictionary.clear();
dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>()) dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
} }
@SneakyThrows
protected static String loadFromClassPath(String path) { protected static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path); URL resource = ResourceLoader.class.getClassLoader().getResource(path);
if (resource == null) { if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
} }
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { List<String> stringList = Files.readAllLines(new File(resource.getPath()).toPath());
StringBuilder sb = new StringBuilder(); return String.join("\n", stringList);
String str;
while ((str = br.readLine()) != null) {
sb.append(str).append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
}
}
protected List<File> getPathsRecursively(File path) {
List<File> result = new ArrayList<>();
if (path == null || path.listFiles() == null) {
return result;
}
for (File f : path.listFiles()) {
if (f.isFile()) {
result.add(f);
} else {
result.addAll(getPathsRecursively(f));
}
}
return result;
} }
protected void loadTypeForTest() { protected void loadTypeForTest() {
typeColorMap.put(VERTEBRATE, "#ff85f7"); typeColorMap.put(VERTEBRATE_INDICATOR, "#ff85f7");
typeColorMap.put(ADDRESS, "#ffe187"); typeColorMap.put(DICTIONARY_ADDRESS, "#ffe187");
typeColorMap.put(AUTHOR, "#ffe187"); typeColorMap.put(DICTIONARY_AUTHOR, "#ffe187");
typeColorMap.put(SPONSOR, "#85ebff"); typeColorMap.put(DICTIONARY_SPONSOR, "#85ebff");
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff"); typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
typeColorMap.put(REDACTION_INDICATOR, "#caff85"); typeColorMap.put(REDACTION_INDICATOR, "#caff85");
typeColorMap.put(HINT_ONLY, "#abc0c4"); typeColorMap.put(HINT_ONLY_INDICATOR, "#abc0c4");
typeColorMap.put(MUST_REDACT, "#fab4c0"); typeColorMap.put(MUST_REDACT_INDICATOR, "#fab4c0");
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff"); typeColorMap.put(PUBLISHED_INFORMATION_INDICATOR, "#85ebff");
typeColorMap.put(TEST_METHOD, "#91fae8"); typeColorMap.put(TEST_METHOD_INDICATOR, "#91fae8");
typeColorMap.put(PII, "#66ccff"); typeColorMap.put(DICTIONARY_PII, "#66ccff");
typeColorMap.put(PURITY, "#ffe187"); typeColorMap.put(PURITY_INDICATOR, "#ffe187");
typeColorMap.put(IMAGE, "#fcc5fb"); typeColorMap.put(IMAGE_INDICATOR, "#fcc5fb");
typeColorMap.put(OCR, "#fcc5fb"); typeColorMap.put(OCR_INDICATOR, "#fcc5fb");
typeColorMap.put(LOGO, "#ffe187"); typeColorMap.put(LOGO_INDICATOR, "#ffe187");
typeColorMap.put(FORMULA, "#ffe187"); typeColorMap.put(FORMULA_INDICATOR, "#ffe187");
typeColorMap.put(SIGNATURE, "#ffe187"); typeColorMap.put(SIGNATURE_INDICATOR, "#ffe187");
typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6"); typeColorMap.put(IMPORTED_REDACTION_INDICATOR, "#fcfbe6");
typeColorMap.put(ROTATE_SIMPLE, "#66ccff"); typeColorMap.put(ROTATE_SIMPLE_INDICATOR, "#66ccff");
hintTypeMap.put(VERTEBRATE, true); hintTypeMap.put(VERTEBRATE_INDICATOR, true);
hintTypeMap.put(ADDRESS, false); hintTypeMap.put(DICTIONARY_ADDRESS, false);
hintTypeMap.put(AUTHOR, false); hintTypeMap.put(DICTIONARY_AUTHOR, false);
hintTypeMap.put(SPONSOR, false); hintTypeMap.put(DICTIONARY_SPONSOR, false);
hintTypeMap.put(NO_REDACTION_INDICATOR, true); hintTypeMap.put(NO_REDACTION_INDICATOR, true);
hintTypeMap.put(REDACTION_INDICATOR, true); hintTypeMap.put(REDACTION_INDICATOR, true);
hintTypeMap.put(HINT_ONLY, true); hintTypeMap.put(HINT_ONLY_INDICATOR, true);
hintTypeMap.put(MUST_REDACT, true); hintTypeMap.put(MUST_REDACT_INDICATOR, true);
hintTypeMap.put(PUBLISHED_INFORMATION, true); hintTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, true);
hintTypeMap.put(TEST_METHOD, true); hintTypeMap.put(TEST_METHOD_INDICATOR, true);
hintTypeMap.put(PII, false); hintTypeMap.put(DICTIONARY_PII, false);
hintTypeMap.put(PURITY, false); hintTypeMap.put(PURITY_INDICATOR, false);
hintTypeMap.put(IMAGE, true); hintTypeMap.put(IMAGE_INDICATOR, true);
hintTypeMap.put(OCR, true); hintTypeMap.put(OCR_INDICATOR, true);
hintTypeMap.put(FORMULA, false); hintTypeMap.put(FORMULA_INDICATOR, false);
hintTypeMap.put(LOGO, false); hintTypeMap.put(LOGO_INDICATOR, false);
hintTypeMap.put(SIGNATURE, false); hintTypeMap.put(SIGNATURE_INDICATOR, false);
hintTypeMap.put(DOSSIER_REDACTIONS, false); hintTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
hintTypeMap.put(IMPORTED_REDACTION, false); hintTypeMap.put(IMPORTED_REDACTION_INDICATOR, false);
hintTypeMap.put(ROTATE_SIMPLE, false); hintTypeMap.put(ROTATE_SIMPLE_INDICATOR, false);
caseInSensitiveMap.put(VERTEBRATE, true); caseInSensitiveMap.put(VERTEBRATE_INDICATOR, true);
caseInSensitiveMap.put(ADDRESS, false); caseInSensitiveMap.put(DICTIONARY_ADDRESS, false);
caseInSensitiveMap.put(AUTHOR, false); caseInSensitiveMap.put(DICTIONARY_AUTHOR, false);
caseInSensitiveMap.put(SPONSOR, false); caseInSensitiveMap.put(DICTIONARY_SPONSOR, false);
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
caseInSensitiveMap.put(REDACTION_INDICATOR, true); caseInSensitiveMap.put(REDACTION_INDICATOR, true);
caseInSensitiveMap.put(HINT_ONLY, true); caseInSensitiveMap.put(HINT_ONLY_INDICATOR, true);
caseInSensitiveMap.put(MUST_REDACT, true); caseInSensitiveMap.put(MUST_REDACT_INDICATOR, true);
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true); caseInSensitiveMap.put(PUBLISHED_INFORMATION_INDICATOR, true);
caseInSensitiveMap.put(TEST_METHOD, false); caseInSensitiveMap.put(TEST_METHOD_INDICATOR, false);
caseInSensitiveMap.put(PII, false); caseInSensitiveMap.put(DICTIONARY_PII, false);
caseInSensitiveMap.put(PURITY, false); caseInSensitiveMap.put(PURITY_INDICATOR, false);
caseInSensitiveMap.put(IMAGE, true); caseInSensitiveMap.put(IMAGE_INDICATOR, true);
caseInSensitiveMap.put(OCR, true); caseInSensitiveMap.put(OCR_INDICATOR, true);
caseInSensitiveMap.put(SIGNATURE, true); caseInSensitiveMap.put(SIGNATURE_INDICATOR, true);
caseInSensitiveMap.put(LOGO, true); caseInSensitiveMap.put(LOGO_INDICATOR, true);
caseInSensitiveMap.put(FORMULA, true); caseInSensitiveMap.put(FORMULA_INDICATOR, true);
caseInSensitiveMap.put(DOSSIER_REDACTIONS, false); caseInSensitiveMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
caseInSensitiveMap.put(IMPORTED_REDACTION, false); caseInSensitiveMap.put(IMPORTED_REDACTION_INDICATOR, false);
caseInSensitiveMap.put(ROTATE_SIMPLE, true); caseInSensitiveMap.put(ROTATE_SIMPLE_INDICATOR, true);
recommendationTypeMap.put(VERTEBRATE, false); recommendationTypeMap.put(VERTEBRATE_INDICATOR, false);
recommendationTypeMap.put(ADDRESS, false); recommendationTypeMap.put(DICTIONARY_ADDRESS, false);
recommendationTypeMap.put(AUTHOR, false); recommendationTypeMap.put(DICTIONARY_AUTHOR, false);
recommendationTypeMap.put(SPONSOR, false); recommendationTypeMap.put(DICTIONARY_SPONSOR, false);
recommendationTypeMap.put(NO_REDACTION_INDICATOR, false); recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
recommendationTypeMap.put(REDACTION_INDICATOR, false); recommendationTypeMap.put(REDACTION_INDICATOR, false);
recommendationTypeMap.put(HINT_ONLY, false); recommendationTypeMap.put(HINT_ONLY_INDICATOR, false);
recommendationTypeMap.put(MUST_REDACT, false); recommendationTypeMap.put(MUST_REDACT_INDICATOR, false);
recommendationTypeMap.put(PUBLISHED_INFORMATION, false); recommendationTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, false);
recommendationTypeMap.put(TEST_METHOD, false); recommendationTypeMap.put(TEST_METHOD_INDICATOR, false);
recommendationTypeMap.put(PII, false); recommendationTypeMap.put(DICTIONARY_PII, false);
recommendationTypeMap.put(PURITY, false); recommendationTypeMap.put(PURITY_INDICATOR, false);
recommendationTypeMap.put(IMAGE, false); recommendationTypeMap.put(IMAGE_INDICATOR, false);
recommendationTypeMap.put(OCR, false); recommendationTypeMap.put(OCR_INDICATOR, false);
recommendationTypeMap.put(FORMULA, false); recommendationTypeMap.put(FORMULA_INDICATOR, false);
recommendationTypeMap.put(SIGNATURE, false); recommendationTypeMap.put(SIGNATURE_INDICATOR, false);
recommendationTypeMap.put(LOGO, false); recommendationTypeMap.put(LOGO_INDICATOR, false);
recommendationTypeMap.put(DOSSIER_REDACTIONS, false); recommendationTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
recommendationTypeMap.put(IMPORTED_REDACTION, false); recommendationTypeMap.put(IMPORTED_REDACTION_INDICATOR, false);
recommendationTypeMap.put(ROTATE_SIMPLE, false); recommendationTypeMap.put(ROTATE_SIMPLE_INDICATOR, false);
rankTypeMap.put(PURITY, 155); rankTypeMap.put(PURITY_INDICATOR, 155);
rankTypeMap.put(PII, 150); rankTypeMap.put(DICTIONARY_PII, 150);
rankTypeMap.put(ADDRESS, 140); rankTypeMap.put(DICTIONARY_ADDRESS, 140);
rankTypeMap.put(AUTHOR, 130); rankTypeMap.put(DICTIONARY_AUTHOR, 130);
rankTypeMap.put(SPONSOR, 120); rankTypeMap.put(DICTIONARY_SPONSOR, 120);
rankTypeMap.put(VERTEBRATE, 110); rankTypeMap.put(VERTEBRATE_INDICATOR, 110);
rankTypeMap.put(MUST_REDACT, 100); rankTypeMap.put(MUST_REDACT_INDICATOR, 100);
rankTypeMap.put(REDACTION_INDICATOR, 90); rankTypeMap.put(REDACTION_INDICATOR, 90);
rankTypeMap.put(NO_REDACTION_INDICATOR, 80); rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
rankTypeMap.put(PUBLISHED_INFORMATION, 70); rankTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, 70);
rankTypeMap.put(TEST_METHOD, 60); rankTypeMap.put(TEST_METHOD_INDICATOR, 60);
rankTypeMap.put(HINT_ONLY, 50); rankTypeMap.put(HINT_ONLY_INDICATOR, 50);
rankTypeMap.put(IMAGE, 30); rankTypeMap.put(IMAGE_INDICATOR, 30);
rankTypeMap.put(OCR, 29); rankTypeMap.put(OCR_INDICATOR, 29);
rankTypeMap.put(LOGO, 28); rankTypeMap.put(LOGO_INDICATOR, 28);
rankTypeMap.put(SIGNATURE, 27); rankTypeMap.put(SIGNATURE_INDICATOR, 27);
rankTypeMap.put(FORMULA, 26); rankTypeMap.put(FORMULA_INDICATOR, 26);
rankTypeMap.put(DOSSIER_REDACTIONS, 200); rankTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, 200);
rankTypeMap.put(IMPORTED_REDACTION, 200); rankTypeMap.put(IMPORTED_REDACTION_INDICATOR, 200);
rankTypeMap.put(ROTATE_SIMPLE, 150); rankTypeMap.put(ROTATE_SIMPLE_INDICATOR, 150);
colors.setSkippedColor("#cccccc"); colors.setSkippedColor("#cccccc");
colors.setRequestAddColor("#04b093"); colors.setRequestAddColor("#04b093");
@ -433,17 +412,19 @@ public abstract class AbstractRedactionIntegrationTest {
if (entries == null) { if (entries == null) {
entries = Collections.emptyList(); entries = Collections.emptyList();
} }
return entries.stream().map(this::toDictionaryEntry).collect(Collectors.toList());
List<DictionaryEntry> dictionaryEntries = new ArrayList<>(); }
entries.forEach(entry -> {
dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
}); private DictionaryEntry toDictionaryEntry(String entry) {
return dictionaryEntries;
return DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build();
} }
@SneakyThrows @SneakyThrows
protected AnalyzeRequest prepareStorage(String file) { protected AnalyzeRequest uploadFileToStorage(String file) {
return prepareStorage(file, "files/cv_service_empty_response.json"); return prepareStorage(file, "files/cv_service_empty_response.json");
} }

121
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -4,12 +4,15 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.util.ArrayList; import java.util.ArrayList;
@ -79,7 +82,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
public static class RedactionIntegrationTestConfiguration { static class RedactionIntegrationTestConfiguration {
@Bean @Bean
public KieContainer kieContainer() { public KieContainer kieContainer() {
@ -123,18 +126,17 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
mockDictionaryCalls(0L);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }
@ -143,7 +145,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void test270Rotated() { public void test270Rotated() {
AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/270Rotated.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
assertThat(result).isNotNull(); assertThat(result).isNotNull();
@ -154,7 +156,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Disabled @Disabled
public void testLargeScannedFileOOM() { public void testLargeScannedFileOOM() {
AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); AnalyzeRequest request = uploadFileToStorage("scanned/VV-377031.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
assertThat(result).isNotNull(); assertThat(result).isNotNull();
@ -166,7 +168,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Minimal Examples/merge_images.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/merge_images.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -180,7 +182,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1)); duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
dictionary.get(AUTHOR).add("Drinking water"); dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
@ -206,7 +208,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
// F. Lastname, J. Doe, M. Mustermann // F. Lastname, J. Doe, M. Mustermann
// Lastname M., Doe J., Mustermann M. // Lastname M., Doe J., Mustermann M.
AnalyzeRequest request = prepareStorage("files/Minimal Examples/ExpansionTest.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/ExpansionTest.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -222,7 +224,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void titleExtraction() throws IOException { public void titleExtraction() throws IOException {
AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -245,7 +247,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void testAddFileAttribute() { public void testAddFileAttribute() {
AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf"); AnalyzeRequest request = uploadFileToStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -262,7 +264,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("testIgnoreHint"); System.out.println("testIgnoreHint");
AnalyzeRequest request = prepareStorage("files/new/test-ignore-hint.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/test-ignore-hint.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);
@ -300,15 +302,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("noExceptionShouldBeThrownForAnyFiles"); System.out.println("noExceptionShouldBeThrownForAnyFiles");
ClassLoader loader = getClass().getClassLoader(); ClassLoader loader = getClass().getClassLoader();
URL url = loader.getResource("files"); URL url = loader.getResource("files");
File[] files = new File(url.getPath()).listFiles(); Path path = Paths.get(URI.create(url.toString()));
List<File> input = new ArrayList<>();
for (File file : files) {
input.addAll(getPathsRecursively(file));
}
for (File path : input) {
AnalyzeRequest request = prepareStorage(path.getPath()); Files.walk(path).forEach(currentPath -> {
System.out.println("Redacting file : " + path.getName()); AnalyzeRequest request = uploadFileToStorage(currentPath.toString());
System.out.println("Redacting file : " + currentPath.getFileName());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
long fstart = System.currentTimeMillis(); long fstart = System.currentTimeMillis();
@ -327,7 +325,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertThat(entry.getValue().size()).isEqualTo(1); assertThat(entry.getValue().size()).isEqualTo(1);
}); });
dictionary.get(AUTHOR).add("Drinking water"); dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
long rstart = System.currentTimeMillis(); long rstart = System.currentTimeMillis();
@ -335,8 +333,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
long rend = System.currentTimeMillis(); long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart)); System.out.println("reanalysis analysis duration: " + (rend - rstart));
});
}
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
@ -352,7 +349,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
request.setExcludedPages(Set.of(1)); request.setExcludedPages(Set.of(1));
request.setFileAttributes(List.of(FileAttribute.builder() request.setFileAttributes(List.of(FileAttribute.builder()
@ -396,10 +393,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
} }
assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
dictionary.get(AUTHOR).add("properties"); dictionary.get(DICTIONARY_AUTHOR).add("properties");
reanlysisVersions.put("properties", 1L); reanlysisVersions.put("properties", 1L);
dictionary.get(AUTHOR).add("physical"); dictionary.get(DICTIONARY_AUTHOR).add("physical");
reanlysisVersions.put("physical", 2L); reanlysisVersions.put("physical", 2L);
deleted.add("David Chubb"); deleted.add("David Chubb");
@ -409,7 +406,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
start = System.currentTimeMillis(); start = System.currentTimeMillis();
@ -442,7 +439,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
@ -458,19 +455,19 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String fileName = "files/new/test1S1T1.pdf"; String fileName = "files/new/test1S1T1.pdf";
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
AnalyzeRequest request = prepareStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);
dictionary.get(AUTHOR).add("report"); dictionary.get(DICTIONARY_AUTHOR).add("report");
reanlysisVersions.put("report", 2L); reanlysisVersions.put("report", 2L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
mockDictionaryCalls(0L); mockDictionaryCalls(0L);
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
dictionary.get(AUTHOR).add("assessment report"); dictionary.get(DICTIONARY_AUTHOR).add("assessment report");
reanlysisVersions.put("assessment report", 3L); reanlysisVersions.put("assessment report", 3L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
mockDictionaryCalls(2L); mockDictionaryCalls(2L);
@ -506,7 +503,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
responseJson.getInputStream()); responseJson.getInputStream());
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
request.setExcludedPages(Set.of(1)); request.setExcludedPages(Set.of(1));
request.setFileAttributes(List.of(FileAttribute.builder() request.setFileAttributes(List.of(FileAttribute.builder()
@ -550,11 +547,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
} }
assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
dictionary.get(AUTHOR).add("properties"); dictionary.get(DICTIONARY_AUTHOR).add("properties");
reanlysisVersions.put("properties", 1L); reanlysisVersions.put("properties", 1L);
mockDictionaryCalls(0L); mockDictionaryCalls(0L);
dictionary.get(AUTHOR).add("physical"); dictionary.get(DICTIONARY_AUTHOR).add("physical");
reanlysisVersions.put("physical", 2L); reanlysisVersions.put("physical", 2L);
deleted.add("David Chubb"); deleted.add("David Chubb");
@ -566,7 +563,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
start = System.currentTimeMillis(); start = System.currentTimeMillis();
@ -600,7 +597,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
@ -623,7 +620,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}; };
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/PublishedInformationTest.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getIdsToRemove() manualRedactions.getIdsToRemove()
@ -681,7 +678,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("testTableRedaction"); System.out.println("testTableRedaction");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -750,7 +747,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/unicodeProblem.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -772,7 +769,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("testTableRedaction"); System.out.println("testTableRedaction");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFile.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -798,7 +795,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("testTableRedaction"); System.out.println("testTableRedaction");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/new/RotateTestFileSimple.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFileSimple.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -820,7 +817,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("testTableHeader"); System.out.println("testTableHeader");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Minimal Examples/NoHeaderTable.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/NoHeaderTable.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -855,7 +852,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/new/S157.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf");
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
@ -931,7 +928,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
AnalyzeRequest request = prepareStorage(pdfFile); AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -964,7 +961,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void phantomCellsDocumentTest() throws IOException { public void phantomCellsDocumentTest() throws IOException {
AnalyzeRequest request = prepareStorage("files/Minimal Examples/Phantom Cells.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/Phantom Cells.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -984,7 +981,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Minimal Examples/sponsor_companies.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/sponsor_companies.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -1046,7 +1043,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry); // manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = prepareStorage(pdfFile); AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -1102,7 +1099,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("expandByRegex"); System.out.println("expandByRegex");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -1157,7 +1154,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
AnalyzeRequest request = prepareStorage(pdfFile); AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -1187,7 +1184,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json"); ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactions.getInputStream()); importedRedactions.getInputStream());
@ -1220,17 +1217,17 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void testExpandByPrefixRegEx() throws IOException { public void testExpandByPrefixRegEx() throws IOException {
assertThat(dictionary.get(AUTHOR)).contains("Robinson"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Robinson");
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mrs. Robinson");
assertThat(dictionary.get(AUTHOR)).contains("Bojangles"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Bojangles");
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Bojangles");
assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Tambourine Man");
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man"); assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Tambourine Man");
String fileName = "files/mr-mrs.pdf"; String fileName = "files/mr-mrs.pdf";
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
AnalyzeRequest request = prepareStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);

31
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationWithRulesV2Test.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTestV2.java
View File

@ -6,7 +6,7 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.Arrays;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@ -40,14 +40,14 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationWithRulesV2Test.RedactionIntegrationTestConfiguration.class) @Import(RedactionIntegrationTestV2.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegrationTest { public class RedactionIntegrationTestV2 extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
public static class RedactionIntegrationTestConfiguration { static class RedactionIntegrationTestConfiguration {
@Bean @Bean
public KieContainer kieContainer() { public KieContainer kieContainer() {
@ -91,18 +91,17 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
mockDictionaryCalls(0L);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }
@ -118,14 +117,14 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr
@SneakyThrows @SneakyThrows
public void testTermIsInTwoDictionariesAndInOneFalsePositive() { public void testTermIsInTwoDictionariesAndInOneFalsePositive() {
AnalyzeRequest request = prepareStorage("files/new/simplified2.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf");
dictionary.clear(); dictionary.clear();
dictionary.computeIfAbsent(PII, v -> new ArrayList<>()).add("Dr. Alan Miller"); dictionary.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller"));
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()).add("Dr. Alan Miller"); dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Dr. Alan Miller"));
falsePositive.clear(); falsePositive.clear();
falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()).add("Dr. Alan Miller COMPLETION DATE:"); falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:"));
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);
@ -136,7 +135,7 @@ public class RedactionIntegrationWithRulesV2Test extends AbstractRedactionIntegr
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo("CBI_author"); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Dr. Alan Miller"); assertThat(redactionLogEntry.getValue()).isEqualTo("Dr. Alan Miller");
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false); assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);