diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java new file mode 100644 index 00000000..6e208b31 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -0,0 +1,503 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; +import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; + +import lombok.SneakyThrows; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(AbstractTestWithDictionaries.TestConfiguration.class) +public class AbstractTestWithDictionaries { + + protected static final String RULES = loadFromClassPath("drools/rules.drl"); + protected static final String RULES_PATH = "drools/rules.drl"; + protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl"; + private static final String VERTEBRATE = "vertebrate"; + private static final String ADDRESS = "CBI_address"; + private static final String AUTHOR = "CBI_author"; + private static final String SPONSOR = "CBI_sponsor"; + private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; + private static final String REDACTION_INDICATOR = "redaction_indicator"; + private static final String HINT_ONLY = "hint_only"; + private static final String MUST_REDACT = "must_redact"; + private static final String PUBLISHED_INFORMATION = "published_information"; + private static final String TEST_METHOD = "test_method"; + private static final String PURITY = "purity"; + private static final String IMAGE = "image"; + private static final String LOGO = "logo"; + private static final String SIGNATURE = "signature"; + private static final String FORMULA = "formula"; + private static final String OCR = "ocr"; + private static final String DOSSIER_REDACTIONS = "dossier_redactions"; + private static final String IMPORTED_REDACTION = "imported_redaction"; + private static final String PII = "PII"; + private static final String ROTATE_SIMPLE = "RotateSimple"; + + protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; + protected final static String TEST_DOSSIER_ID = "123"; + protected final static String TEST_FILE_ID = "123"; + + @Autowired + protected StorageService storageService; + + @MockBean + protected DictionaryClient dictionaryClient; + + @MockBean + protected RulesClient rulesClient; + + private final Map> dictionary = new HashMap<>(); + private final Map> dossierDictionary = new HashMap<>(); + private final Map> falsePositive = new HashMap<>(); + private final Map> falseRecommendation = new HashMap<>(); + private final Map typeColorMap = new HashMap<>(); + private final Map hintTypeMap = new HashMap<>(); + private final Map caseInSensitiveMap = new HashMap<>(); + private final Map recommendationTypeMap = new HashMap<>(); + private final Map rankTypeMap = new HashMap<>(); + private final Colors colors = new Colors(); + private final Map reanalysisVersions = new HashMap<>(); + private final Set deleted = new HashSet<>(); + + + @BeforeEach + public void stubClients() { + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) + .build())); + + mockDictionaryCalls(null); + mockDictionaryCalls(0L); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + private static String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader().getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str).append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + } + + + private void mockDictionaryCalls(Long version) { + + when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, + false)); + when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + NO_REDACTION_INDICATOR, + false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + REDACTION_INDICATOR, + false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT, + false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + PUBLISHED_INFORMATION, + false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD, + false)); + when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PII, false)); + when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY, false)); + when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE, false)); + when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(OCR, false)); + when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE, false)); + when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA, false)); + when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE, + false)); + when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + DOSSIER_REDACTIONS, + true)); + when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + IMPORTED_REDACTION, + true)); + + } + + + private String cleanDictionaryEntry(String entry) { + + return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); + } + + + private List toDictionaryEntry(List entries) { + + if (entries == null) { + entries = Collections.emptyList(); + } + + List dictionaryEntries = new ArrayList<>(); + entries.forEach(entry -> { + dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build()); + }); + return dictionaryEntries; + } + + + private Type getDictionaryResponse(String type, boolean isDossierDictionary) { + + return Type.builder() + .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(type)) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) + .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>()) + .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>()) + .isHint(hintTypeMap.get(type)) + .isCaseInsensitive(caseInSensitiveMap.get(type)) + .isRecommendation(recommendationTypeMap.get(type)) + .rank(rankTypeMap.get(type)) + .build(); + } + + + private void loadTypeForTest() { + + typeColorMap.put(VERTEBRATE, "#ff85f7"); + typeColorMap.put(ADDRESS, "#ffe187"); + typeColorMap.put(AUTHOR, "#ffe187"); + typeColorMap.put(SPONSOR, "#85ebff"); + typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff"); + typeColorMap.put(REDACTION_INDICATOR, "#caff85"); + typeColorMap.put(HINT_ONLY, "#abc0c4"); + typeColorMap.put(MUST_REDACT, "#fab4c0"); + typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff"); + typeColorMap.put(TEST_METHOD, "#91fae8"); + typeColorMap.put(PII, "#66ccff"); + typeColorMap.put(PURITY, "#ffe187"); + typeColorMap.put(IMAGE, "#fcc5fb"); + typeColorMap.put(OCR, "#fcc5fb"); + typeColorMap.put(LOGO, "#ffe187"); + typeColorMap.put(FORMULA, "#ffe187"); + typeColorMap.put(SIGNATURE, "#ffe187"); + typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6"); + typeColorMap.put(ROTATE_SIMPLE, "#66ccff"); + + hintTypeMap.put(VERTEBRATE, true); + hintTypeMap.put(ADDRESS, false); + hintTypeMap.put(AUTHOR, false); + hintTypeMap.put(SPONSOR, false); + hintTypeMap.put(NO_REDACTION_INDICATOR, true); + hintTypeMap.put(REDACTION_INDICATOR, true); + hintTypeMap.put(HINT_ONLY, true); + hintTypeMap.put(MUST_REDACT, true); + hintTypeMap.put(PUBLISHED_INFORMATION, true); + hintTypeMap.put(TEST_METHOD, true); + hintTypeMap.put(PII, false); + hintTypeMap.put(PURITY, false); + hintTypeMap.put(IMAGE, true); + hintTypeMap.put(OCR, true); + hintTypeMap.put(FORMULA, false); + hintTypeMap.put(LOGO, false); + hintTypeMap.put(SIGNATURE, false); + hintTypeMap.put(DOSSIER_REDACTIONS, false); + hintTypeMap.put(IMPORTED_REDACTION, false); + hintTypeMap.put(ROTATE_SIMPLE, false); + + caseInSensitiveMap.put(VERTEBRATE, true); + caseInSensitiveMap.put(ADDRESS, false); + caseInSensitiveMap.put(AUTHOR, false); + caseInSensitiveMap.put(SPONSOR, false); + caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); + caseInSensitiveMap.put(REDACTION_INDICATOR, true); + caseInSensitiveMap.put(HINT_ONLY, true); + caseInSensitiveMap.put(MUST_REDACT, true); + caseInSensitiveMap.put(PUBLISHED_INFORMATION, true); + caseInSensitiveMap.put(TEST_METHOD, false); + caseInSensitiveMap.put(PII, false); + caseInSensitiveMap.put(PURITY, false); + caseInSensitiveMap.put(IMAGE, true); + caseInSensitiveMap.put(OCR, true); + caseInSensitiveMap.put(SIGNATURE, true); + caseInSensitiveMap.put(LOGO, true); + caseInSensitiveMap.put(FORMULA, true); + caseInSensitiveMap.put(DOSSIER_REDACTIONS, false); + caseInSensitiveMap.put(IMPORTED_REDACTION, false); + caseInSensitiveMap.put(ROTATE_SIMPLE, true); + + recommendationTypeMap.put(VERTEBRATE, false); + recommendationTypeMap.put(ADDRESS, false); + recommendationTypeMap.put(AUTHOR, false); + recommendationTypeMap.put(SPONSOR, false); + recommendationTypeMap.put(NO_REDACTION_INDICATOR, false); + recommendationTypeMap.put(REDACTION_INDICATOR, false); + recommendationTypeMap.put(HINT_ONLY, false); + recommendationTypeMap.put(MUST_REDACT, false); + recommendationTypeMap.put(PUBLISHED_INFORMATION, false); + recommendationTypeMap.put(TEST_METHOD, false); + recommendationTypeMap.put(PII, false); + recommendationTypeMap.put(PURITY, false); + recommendationTypeMap.put(IMAGE, false); + recommendationTypeMap.put(OCR, false); + recommendationTypeMap.put(FORMULA, false); + recommendationTypeMap.put(SIGNATURE, false); + recommendationTypeMap.put(LOGO, false); + recommendationTypeMap.put(DOSSIER_REDACTIONS, false); + recommendationTypeMap.put(IMPORTED_REDACTION, false); + recommendationTypeMap.put(ROTATE_SIMPLE, false); + + rankTypeMap.put(PURITY, 155); + rankTypeMap.put(PII, 150); + rankTypeMap.put(ADDRESS, 140); + rankTypeMap.put(AUTHOR, 130); + rankTypeMap.put(SPONSOR, 120); + rankTypeMap.put(VERTEBRATE, 110); + rankTypeMap.put(MUST_REDACT, 100); + rankTypeMap.put(REDACTION_INDICATOR, 90); + rankTypeMap.put(NO_REDACTION_INDICATOR, 80); + rankTypeMap.put(PUBLISHED_INFORMATION, 70); + rankTypeMap.put(TEST_METHOD, 60); + rankTypeMap.put(HINT_ONLY, 50); + rankTypeMap.put(IMAGE, 30); + rankTypeMap.put(OCR, 29); + rankTypeMap.put(LOGO, 28); + rankTypeMap.put(SIGNATURE, 27); + rankTypeMap.put(FORMULA, 26); + rankTypeMap.put(DOSSIER_REDACTIONS, 200); + rankTypeMap.put(IMPORTED_REDACTION, 200); + rankTypeMap.put(ROTATE_SIMPLE, 150); + + colors.setSkippedColor("#cccccc"); + colors.setRequestAddColor("#04b093"); + colors.setRequestRemoveColor("#04b093"); + } + + + @SneakyThrows + private void loadNerForTest() { + + ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + } + + + private List getTypeResponse() { + + return typeColorMap.entrySet() + .stream() + .map(typeColor -> Type.builder() + .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(typeColor.getKey()) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColor.getValue()) + .isHint(hintTypeMap.get(typeColor.getKey())) + .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) + .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) + .rank(rankTypeMap.get(typeColor.getKey())) + .build()) + + .collect(Collectors.toList()); + } + + + protected List getPathsRecursively(File path) { + + List result = new ArrayList<>(); + if (path == null || path.listFiles() == null) { + return result; + } + for (File f : path.listFiles()) { + if (f.isFile()) { + result.add(f); + } else { + result.addAll(getPathsRecursively(f)); + } + } + return result; + + } + + + protected void loadOnlyDictionaryForSimpleFile() { + + dictionary.clear(); + dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + } + + + private void loadDictionaryForTest() { + + dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); + + falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(String file) { + + return prepareStorage(file, "files/cv_service_empty_response.json"); + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) { + + ClassPathResource pdfFileResource = new ClassPathResource(file); + ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile); + + return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream()); + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) { + + AnalyzeRequest request = AnalyzeRequest.builder() + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .lastProcessed(OffsetDateTime.now()) + .build(); + + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); + + return request; + } + + + @AfterEach + public void cleanupStorage() { + + if (this.storageService instanceof FileSystemBackedStorageService) { + ((FileSystemBackedStorageService) this.storageService).clearStorage(); + } + } + + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) + public static class TestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(); + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisIntegrationTest.java new file mode 100644 index 00000000..a5478d1d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisIntegrationTest.java @@ -0,0 +1,213 @@ +package com.iqser.red.service.redaction.v1.server; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.servlet.ServletContext; + +import org.apache.commons.collections4.SetUtils; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; + +import com.google.common.io.Files; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; +import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.Builder; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AnalysisIntegrationTest extends AbstractTestWithDictionaries { + + @Autowired + private AnalyzeService analyzeService; + + @Autowired + private ServletContext context; + + @Autowired + private RedactionStorageService redactionStorageService; + + @MockBean + private RabbitTemplate rabbitTemplate; + + @MockBean + private LegalBasisClient legalBasisClient; + + + @Test + public void testAnalysisIntegrationForAllFiles() { + + findAllFilesWithRedactionLogs().forEach((pdfFile, redactionLogFile) -> { + + try { + String relativePdfFilePath = new ClassPathResource("").getURI().relativize(pdfFile.toURI()).getPath(); + log.info("Checking equality for {}", relativePdfFilePath); + AnalyzeRequest request = prepareStorage(relativePdfFilePath); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); + + RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + Set references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet()); + + Path entityReferencePath = Paths.get(new ClassPathResource("entities/").getFile().getAbsolutePath().replace("target/test-classes/", "src/test/resources/"), + relativePdfFilePath.replace("files/", "").replace(".pdf", ".json")); + + new File(entityReferencePath.toString().replace(entityReferencePath.getFileName().toString(), "")).mkdirs(); + + try (var out = new FileOutputStream(entityReferencePath.toString())) { + ObjectMapperFactory.create().writeValue(out, references); + log.info("{} Entities have been written for File {}", references.size(), relativePdfFilePath); + } + //RedactionLog originalRedactionLog = ObjectMapperFactory.create().readValue(redactionLogFile, RedactionLog.class); + //assertEntitiesAreEqual(originalRedactionLog, redactionLog); + } catch (IOException e) { + throw new RuntimeException(e); + } + + }); + } + + + // only use this if you must replace all entities with newer versions + @Test + @Disabled + @SneakyThrows + public void writeNewEntitiesForAllFiles() { + + ClassPathResource fileStem = new ClassPathResource("files/"); + List allPdfFiles = getAllPdfFilesAsStream(fileStem).toList(); + System.out.printf("found %d PDF Files to create Entities for\n", allPdfFiles.size()); + int count = 0; + for (File pdfFile : allPdfFiles) { + writeNewEntities(pdfFile); + System.out.printf("%d|%d finished writing entities for %s\n", count, allPdfFiles.size(), pdfFile.getName()); + count++; + } + } + + + private static Stream getAllPdfFilesAsStream(ClassPathResource fileStem) throws IOException { + + return Arrays.stream(fileStem.getFile().listFiles()) + .filter(File::isDirectory) + .flatMap(file -> Arrays.stream(file.listFiles())) + .filter(file -> Files.getFileExtension(file.getName()).equals("pdf")) + .filter(File::isFile) + .filter(File::canRead); + } + + + @SneakyThrows + private void writeNewEntities(File pdfFile) { + + Path relativePdfFilePath = Paths.get(pdfFile.toString().replace(new ClassPathResource("").getFile().getAbsolutePath(), "")); + Path relativeFolderPath = relativePdfFilePath.getParent(); + Path absoluteEntityFolderPath = Paths.get(context.getContextPath(), "src", "test", "resources", "entities", relativeFolderPath.toString().replace("files/", "")); + absoluteEntityFolderPath.toFile().mkdirs(); + Path absoluteEntityFilePath = absoluteEntityFolderPath.resolve(relativePdfFilePath.getFileName().toString().replace("files/", "").replace(".pdf", ".json")); + + AnalyzeRequest request = prepareStorage(relativePdfFilePath.toString()); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); + + RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + Set references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet()); + + try (var out = new FileOutputStream(absoluteEntityFilePath.toString())) { + ObjectMapperFactory.create().writeValue(out, references); + } + + } + + + private void assertEntitiesAreEqual(RedactionLog originalRedactionLog, RedactionLog redactionLog) { + + Set originalReferences = originalRedactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet()); + Set references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet()); + + Set missingReferences = SetUtils.difference(originalReferences, references); + Set additionalReferences = SetUtils.difference(references, originalReferences); + + if (missingReferences.size() == 0) { + log.info("All original Entities have been found!"); + } else { + log.warn("{} original References not found {}", missingReferences.size(), missingReferences); + assert missingReferences.size() == 0; + } + + if (additionalReferences.size() == 0) { + log.info("No additional Entities have been found!"); + } else { + log.warn("{} additional Entities found", additionalReferences.size()); + additionalReferences.forEach(e -> log.warn(e.toString())); + assert additionalReferences.size() == 0; + } + } + + + private EntityReference toReference(RedactionLogEntry redactionLogEntry) { + + return EntityReference.builder() + .positions(redactionLogEntry.getPositions()) + .isRecommendation(redactionLogEntry.isRecommendation()) + .isFalsePositive(redactionLogEntry.isFalsePositive()) + .word(redactionLogEntry.getValue()) + .isHint(redactionLogEntry.isHint()) + .isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) + .reason(redactionLogEntry.getReason()) + .build(); + } + + + @Builder + public record EntityReference( + String word, String reason, boolean redacted, boolean isHint, boolean isRecommendation, boolean isFalsePositive, boolean isDictionaryEntry, List positions) { + + } + + + @SneakyThrows + private Map findAllFilesWithRedactionLogs() { + + ClassPathResource fileStem = new ClassPathResource("files/"); + ClassPathResource redactionStem = new ClassPathResource("RedactionLog/files/"); + + Map fileToRedactionLogFile = new HashMap<>(); + List allFiles = getAllPdfFilesAsStream(fileStem).toList(); + + for (File file : allFiles) { + String relativePath = fileStem.getURI().relativize(file.toURI()).getPath().replace(".pdf", ".json"); + File redactionLogFile = Paths.get(redactionStem.getFile().getAbsolutePath(), relativePath).toFile(); + if (redactionLogFile.isFile() && redactionLogFile.canRead()) { + fileToRedactionLogFile.put(file, redactionLogFile); + } + } + return fileToRedactionLogFile; + } + +}