RED-6093: Prototype implementation for document structure and analysis concept
*added a new analysis test to ensure the same entities are being found
This commit is contained in:
Kilian Schuettler
parent
cab54f6cec
commit
1cd6b73d9a
@ -0,0 +1,503 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.stubbing.Answer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(AbstractTestWithDictionaries.TestConfiguration.class)
|
||||
public class AbstractTestWithDictionaries {
|
||||
|
||||
protected static final String RULES = loadFromClassPath("drools/rules.drl");
|
||||
protected static final String RULES_PATH = "drools/rules.drl";
|
||||
protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
|
||||
private static final String VERTEBRATE = "vertebrate";
|
||||
private static final String ADDRESS = "CBI_address";
|
||||
private static final String AUTHOR = "CBI_author";
|
||||
private static final String SPONSOR = "CBI_sponsor";
|
||||
private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
|
||||
private static final String REDACTION_INDICATOR = "redaction_indicator";
|
||||
private static final String HINT_ONLY = "hint_only";
|
||||
private static final String MUST_REDACT = "must_redact";
|
||||
private static final String PUBLISHED_INFORMATION = "published_information";
|
||||
private static final String TEST_METHOD = "test_method";
|
||||
private static final String PURITY = "purity";
|
||||
private static final String IMAGE = "image";
|
||||
private static final String LOGO = "logo";
|
||||
private static final String SIGNATURE = "signature";
|
||||
private static final String FORMULA = "formula";
|
||||
private static final String OCR = "ocr";
|
||||
private static final String DOSSIER_REDACTIONS = "dossier_redactions";
|
||||
private static final String IMPORTED_REDACTION = "imported_redaction";
|
||||
private static final String PII = "PII";
|
||||
private static final String ROTATE_SIMPLE = "RotateSimple";
|
||||
|
||||
protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
|
||||
protected final static String TEST_DOSSIER_ID = "123";
|
||||
protected final static String TEST_FILE_ID = "123";
|
||||
|
||||
@Autowired
|
||||
protected StorageService storageService;
|
||||
|
||||
@MockBean
|
||||
protected DictionaryClient dictionaryClient;
|
||||
|
||||
@MockBean
|
||||
protected RulesClient rulesClient;
|
||||
|
||||
private final Map<String, List<String>> dictionary = new HashMap<>();
|
||||
private final Map<String, List<String>> dossierDictionary = new HashMap<>();
|
||||
private final Map<String, List<String>> falsePositive = new HashMap<>();
|
||||
private final Map<String, List<String>> falseRecommendation = new HashMap<>();
|
||||
private final Map<String, String> typeColorMap = new HashMap<>();
|
||||
private final Map<String, Boolean> hintTypeMap = new HashMap<>();
|
||||
private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
|
||||
private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
|
||||
private final Map<String, Integer> rankTypeMap = new HashMap<>();
|
||||
private final Colors colors = new Colors();
|
||||
private final Map<String, Long> reanalysisVersions = new HashMap<>();
|
||||
private final Set<String> deleted = new HashSet<>();
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void stubClients() {
|
||||
|
||||
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
|
||||
|
||||
loadDictionaryForTest();
|
||||
loadTypeForTest();
|
||||
loadNerForTest();
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
|
||||
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(DOSSIER_REDACTIONS)
|
||||
.dossierTemplateId(TEST_DOSSIER_ID)
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
|
||||
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void mockDictionaryCalls(Long version) {
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
NO_REDACTION_INDICATOR,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
REDACTION_INDICATOR,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
PUBLISHED_INFORMATION,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
|
||||
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
|
||||
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
|
||||
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
|
||||
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
DOSSIER_REDACTIONS,
|
||||
true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
IMPORTED_REDACTION,
|
||||
true));
|
||||
|
||||
}
|
||||
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
|
||||
}
|
||||
|
||||
|
||||
private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
|
||||
|
||||
if (entries == null) {
|
||||
entries = Collections.emptyList();
|
||||
}
|
||||
|
||||
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
|
||||
entries.forEach(entry -> {
|
||||
dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
|
||||
});
|
||||
return dictionaryEntries;
|
||||
}
|
||||
|
||||
|
||||
private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
|
||||
|
||||
return Type.builder()
|
||||
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColorMap.get(type))
|
||||
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
|
||||
.falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
|
||||
.falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
|
||||
.isHint(hintTypeMap.get(type))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(type))
|
||||
.isRecommendation(recommendationTypeMap.get(type))
|
||||
.rank(rankTypeMap.get(type))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private void loadTypeForTest() {
|
||||
|
||||
typeColorMap.put(VERTEBRATE, "#ff85f7");
|
||||
typeColorMap.put(ADDRESS, "#ffe187");
|
||||
typeColorMap.put(AUTHOR, "#ffe187");
|
||||
typeColorMap.put(SPONSOR, "#85ebff");
|
||||
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
|
||||
typeColorMap.put(REDACTION_INDICATOR, "#caff85");
|
||||
typeColorMap.put(HINT_ONLY, "#abc0c4");
|
||||
typeColorMap.put(MUST_REDACT, "#fab4c0");
|
||||
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
|
||||
typeColorMap.put(TEST_METHOD, "#91fae8");
|
||||
typeColorMap.put(PII, "#66ccff");
|
||||
typeColorMap.put(PURITY, "#ffe187");
|
||||
typeColorMap.put(IMAGE, "#fcc5fb");
|
||||
typeColorMap.put(OCR, "#fcc5fb");
|
||||
typeColorMap.put(LOGO, "#ffe187");
|
||||
typeColorMap.put(FORMULA, "#ffe187");
|
||||
typeColorMap.put(SIGNATURE, "#ffe187");
|
||||
typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
|
||||
typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
hintTypeMap.put(AUTHOR, false);
|
||||
hintTypeMap.put(SPONSOR, false);
|
||||
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(HINT_ONLY, true);
|
||||
hintTypeMap.put(MUST_REDACT, true);
|
||||
hintTypeMap.put(PUBLISHED_INFORMATION, true);
|
||||
hintTypeMap.put(TEST_METHOD, true);
|
||||
hintTypeMap.put(PII, false);
|
||||
hintTypeMap.put(PURITY, false);
|
||||
hintTypeMap.put(IMAGE, true);
|
||||
hintTypeMap.put(OCR, true);
|
||||
hintTypeMap.put(FORMULA, false);
|
||||
hintTypeMap.put(LOGO, false);
|
||||
hintTypeMap.put(SIGNATURE, false);
|
||||
hintTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
hintTypeMap.put(IMPORTED_REDACTION, false);
|
||||
hintTypeMap.put(ROTATE_SIMPLE, false);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
caseInSensitiveMap.put(AUTHOR, false);
|
||||
caseInSensitiveMap.put(SPONSOR, false);
|
||||
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(HINT_ONLY, true);
|
||||
caseInSensitiveMap.put(MUST_REDACT, true);
|
||||
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
|
||||
caseInSensitiveMap.put(TEST_METHOD, false);
|
||||
caseInSensitiveMap.put(PII, false);
|
||||
caseInSensitiveMap.put(PURITY, false);
|
||||
caseInSensitiveMap.put(IMAGE, true);
|
||||
caseInSensitiveMap.put(OCR, true);
|
||||
caseInSensitiveMap.put(SIGNATURE, true);
|
||||
caseInSensitiveMap.put(LOGO, true);
|
||||
caseInSensitiveMap.put(FORMULA, true);
|
||||
caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
|
||||
caseInSensitiveMap.put(IMPORTED_REDACTION, false);
|
||||
caseInSensitiveMap.put(ROTATE_SIMPLE, true);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
recommendationTypeMap.put(AUTHOR, false);
|
||||
recommendationTypeMap.put(SPONSOR, false);
|
||||
recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(HINT_ONLY, false);
|
||||
recommendationTypeMap.put(MUST_REDACT, false);
|
||||
recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
|
||||
recommendationTypeMap.put(TEST_METHOD, false);
|
||||
recommendationTypeMap.put(PII, false);
|
||||
recommendationTypeMap.put(PURITY, false);
|
||||
recommendationTypeMap.put(IMAGE, false);
|
||||
recommendationTypeMap.put(OCR, false);
|
||||
recommendationTypeMap.put(FORMULA, false);
|
||||
recommendationTypeMap.put(SIGNATURE, false);
|
||||
recommendationTypeMap.put(LOGO, false);
|
||||
recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
recommendationTypeMap.put(IMPORTED_REDACTION, false);
|
||||
recommendationTypeMap.put(ROTATE_SIMPLE, false);
|
||||
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
rankTypeMap.put(PII, 150);
|
||||
rankTypeMap.put(ADDRESS, 140);
|
||||
rankTypeMap.put(AUTHOR, 130);
|
||||
rankTypeMap.put(SPONSOR, 120);
|
||||
rankTypeMap.put(VERTEBRATE, 110);
|
||||
rankTypeMap.put(MUST_REDACT, 100);
|
||||
rankTypeMap.put(REDACTION_INDICATOR, 90);
|
||||
rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
|
||||
rankTypeMap.put(PUBLISHED_INFORMATION, 70);
|
||||
rankTypeMap.put(TEST_METHOD, 60);
|
||||
rankTypeMap.put(HINT_ONLY, 50);
|
||||
rankTypeMap.put(IMAGE, 30);
|
||||
rankTypeMap.put(OCR, 29);
|
||||
rankTypeMap.put(LOGO, 28);
|
||||
rankTypeMap.put(SIGNATURE, 27);
|
||||
rankTypeMap.put(FORMULA, 26);
|
||||
rankTypeMap.put(DOSSIER_REDACTIONS, 200);
|
||||
rankTypeMap.put(IMPORTED_REDACTION, 200);
|
||||
rankTypeMap.put(ROTATE_SIMPLE, 150);
|
||||
|
||||
colors.setSkippedColor("#cccccc");
|
||||
colors.setRequestAddColor("#04b093");
|
||||
colors.setRequestRemoveColor("#04b093");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getTypeResponse() {
|
||||
|
||||
return typeColorMap.entrySet()
|
||||
.stream()
|
||||
.map(typeColor -> Type.builder()
|
||||
.id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(typeColor.getKey())
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
.build())
|
||||
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
protected List<File> getPathsRecursively(File path) {
|
||||
|
||||
List<File> result = new ArrayList<>();
|
||||
if (path == null || path.listFiles() == null) {
|
||||
return result;
|
||||
}
|
||||
for (File f : path.listFiles()) {
|
||||
if (f.isFile()) {
|
||||
result.add(f);
|
||||
} else {
|
||||
result.addAll(getPathsRecursively(f));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
|
||||
protected void loadOnlyDictionaryForSimpleFile() {
|
||||
|
||||
dictionary.clear();
|
||||
dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
|
||||
|
||||
falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
protected AnalyzeRequest prepareStorage(String file) {
|
||||
|
||||
return prepareStorage(file, "files/cv_service_empty_response.json");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||
ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
|
||||
|
||||
return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
|
||||
@AfterEach
|
||||
public void cleanupStorage() {
|
||||
|
||||
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
|
||||
public static class TestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
|
||||
return new FileSystemBackedStorageService();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,213 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import javax.servlet.ServletContext;
|
||||
|
||||
import org.apache.commons.collections4.SetUtils;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import com.google.common.io.Files;
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class AnalysisIntegrationTest extends AbstractTestWithDictionaries {
|
||||
|
||||
@Autowired
|
||||
private AnalyzeService analyzeService;
|
||||
|
||||
@Autowired
|
||||
private ServletContext context;
|
||||
|
||||
@Autowired
|
||||
private RedactionStorageService redactionStorageService;
|
||||
|
||||
@MockBean
|
||||
private RabbitTemplate rabbitTemplate;
|
||||
|
||||
@MockBean
|
||||
private LegalBasisClient legalBasisClient;
|
||||
|
||||
|
||||
@Test
|
||||
public void testAnalysisIntegrationForAllFiles() {
|
||||
|
||||
findAllFilesWithRedactionLogs().forEach((pdfFile, redactionLogFile) -> {
|
||||
|
||||
try {
|
||||
String relativePdfFilePath = new ClassPathResource("").getURI().relativize(pdfFile.toURI()).getPath();
|
||||
log.info("Checking equality for {}", relativePdfFilePath);
|
||||
AnalyzeRequest request = prepareStorage(relativePdfFilePath);
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
|
||||
|
||||
Path entityReferencePath = Paths.get(new ClassPathResource("entities/").getFile().getAbsolutePath().replace("target/test-classes/", "src/test/resources/"),
|
||||
relativePdfFilePath.replace("files/", "").replace(".pdf", ".json"));
|
||||
|
||||
new File(entityReferencePath.toString().replace(entityReferencePath.getFileName().toString(), "")).mkdirs();
|
||||
|
||||
try (var out = new FileOutputStream(entityReferencePath.toString())) {
|
||||
ObjectMapperFactory.create().writeValue(out, references);
|
||||
log.info("{} Entities have been written for File {}", references.size(), relativePdfFilePath);
|
||||
}
|
||||
//RedactionLog originalRedactionLog = ObjectMapperFactory.create().readValue(redactionLogFile, RedactionLog.class);
|
||||
//assertEntitiesAreEqual(originalRedactionLog, redactionLog);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// only use this if you must replace all entities with newer versions
|
||||
@Test
|
||||
@Disabled
|
||||
@SneakyThrows
|
||||
public void writeNewEntitiesForAllFiles() {
|
||||
|
||||
ClassPathResource fileStem = new ClassPathResource("files/");
|
||||
List<File> allPdfFiles = getAllPdfFilesAsStream(fileStem).toList();
|
||||
System.out.printf("found %d PDF Files to create Entities for\n", allPdfFiles.size());
|
||||
int count = 0;
|
||||
for (File pdfFile : allPdfFiles) {
|
||||
writeNewEntities(pdfFile);
|
||||
System.out.printf("%d|%d finished writing entities for %s\n", count, allPdfFiles.size(), pdfFile.getName());
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static Stream<File> getAllPdfFilesAsStream(ClassPathResource fileStem) throws IOException {
|
||||
|
||||
return Arrays.stream(fileStem.getFile().listFiles())
|
||||
.filter(File::isDirectory)
|
||||
.flatMap(file -> Arrays.stream(file.listFiles()))
|
||||
.filter(file -> Files.getFileExtension(file.getName()).equals("pdf"))
|
||||
.filter(File::isFile)
|
||||
.filter(File::canRead);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void writeNewEntities(File pdfFile) {
|
||||
|
||||
Path relativePdfFilePath = Paths.get(pdfFile.toString().replace(new ClassPathResource("").getFile().getAbsolutePath(), ""));
|
||||
Path relativeFolderPath = relativePdfFilePath.getParent();
|
||||
Path absoluteEntityFolderPath = Paths.get(context.getContextPath(), "src", "test", "resources", "entities", relativeFolderPath.toString().replace("files/", ""));
|
||||
absoluteEntityFolderPath.toFile().mkdirs();
|
||||
Path absoluteEntityFilePath = absoluteEntityFolderPath.resolve(relativePdfFilePath.getFileName().toString().replace("files/", "").replace(".pdf", ".json"));
|
||||
|
||||
AnalyzeRequest request = prepareStorage(relativePdfFilePath.toString());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
|
||||
|
||||
try (var out = new FileOutputStream(absoluteEntityFilePath.toString())) {
|
||||
ObjectMapperFactory.create().writeValue(out, references);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void assertEntitiesAreEqual(RedactionLog originalRedactionLog, RedactionLog redactionLog) {
|
||||
|
||||
Set<EntityReference> originalReferences = originalRedactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
|
||||
Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
|
||||
|
||||
Set<EntityReference> missingReferences = SetUtils.difference(originalReferences, references);
|
||||
Set<EntityReference> additionalReferences = SetUtils.difference(references, originalReferences);
|
||||
|
||||
if (missingReferences.size() == 0) {
|
||||
log.info("All original Entities have been found!");
|
||||
} else {
|
||||
log.warn("{} original References not found {}", missingReferences.size(), missingReferences);
|
||||
assert missingReferences.size() == 0;
|
||||
}
|
||||
|
||||
if (additionalReferences.size() == 0) {
|
||||
log.info("No additional Entities have been found!");
|
||||
} else {
|
||||
log.warn("{} additional Entities found", additionalReferences.size());
|
||||
additionalReferences.forEach(e -> log.warn(e.toString()));
|
||||
assert additionalReferences.size() == 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private EntityReference toReference(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
return EntityReference.builder()
|
||||
.positions(redactionLogEntry.getPositions())
|
||||
.isRecommendation(redactionLogEntry.isRecommendation())
|
||||
.isFalsePositive(redactionLogEntry.isFalsePositive())
|
||||
.word(redactionLogEntry.getValue())
|
||||
.isHint(redactionLogEntry.isHint())
|
||||
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
|
||||
.reason(redactionLogEntry.getReason())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Builder
|
||||
public record EntityReference(
|
||||
String word, String reason, boolean redacted, boolean isHint, boolean isRecommendation, boolean isFalsePositive, boolean isDictionaryEntry, List<Rectangle> positions) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private Map<File, File> findAllFilesWithRedactionLogs() {
|
||||
|
||||
ClassPathResource fileStem = new ClassPathResource("files/");
|
||||
ClassPathResource redactionStem = new ClassPathResource("RedactionLog/files/");
|
||||
|
||||
Map<File, File> fileToRedactionLogFile = new HashMap<>();
|
||||
List<File> allFiles = getAllPdfFilesAsStream(fileStem).toList();
|
||||
|
||||
for (File file : allFiles) {
|
||||
String relativePath = fileStem.getURI().relativize(file.toURI()).getPath().replace(".pdf", ".json");
|
||||
File redactionLogFile = Paths.get(redactionStem.getFile().getAbsolutePath(), relativePath).toFile();
|
||||
if (redactionLogFile.isFile() && redactionLogFile.canRead()) {
|
||||
fileToRedactionLogFile.put(file, redactionLogFile);
|
||||
}
|
||||
}
|
||||
return fileToRedactionLogFile;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user