Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-9187-junit' into 'master'

Browse Source 
RED-9187 - Annotation does not switch back to redacted if published...

Closes RED-9187

See merge request redactmanager/redaction-service!425
This commit is contained in:
Corina Olariu 2024-06-13 08:36:57 +02:00
commit c2acb90080
4 changed files with 107 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -74,6 +74,12 @@ public class DictionaryService {
}
public void clearTenantDictionaryCache() {
tenantDictionaryCache.invalidateAll();
}
@SneakyThrows
@Observed(name = "DictionaryService", contextualName = "update-dictionary")
@Timed("redactmanager_updateDictionary")

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -100,6 +100,8 @@ public abstract class AbstractRedactionIntegrationTest {
protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple";
protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
protected final static String TEST_DOSSIER_ID = "123";
protected final static String TEST_FILE_ID = "123";
public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
@ -112,6 +114,7 @@ public abstract class AbstractRedactionIntegrationTest {
public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID+ ":" + TEST_DOSSIER_ID;
public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
@ -181,8 +184,6 @@ public abstract class AbstractRedactionIntegrationTest {
protected final Map<String, Long> reanlysisVersions = new HashMap<>();
protected final Set<String> deleted = new HashSet<>();
protected final static String TEST_DOSSIER_ID = "123";
protected final static String TEST_FILE_ID = "123";
@MockBean
protected RulesClient rulesClient;
@ -249,6 +250,8 @@ public abstract class AbstractRedactionIntegrationTest {
true));
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
true));
when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
true));
}
@ -346,6 +349,7 @@ public abstract class AbstractRedactionIntegrationTest {
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")

98
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -7,6 +7,7 @@ import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
@ -14,6 +15,7 @@ import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
@ -40,6 +42,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@ -54,6 +57,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@Autowired
DictionaryService dictionaryService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class)
@ -87,14 +93,26 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.id(DOSSIER_REDACTIONS_TYPE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build(),
Type.builder()
.id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
.type(PUBLISHED_INFORMATION_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe180")
.isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.build()));
mockDictionaryCalls(null);
@ -128,6 +146,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
dictionaryService.clearTenantDictionaryCache();
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@ -141,7 +161,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
.orElseThrow();
assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
assertThat(asyaLyon1.getSection().startsWith("Paragraph:"));
assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@ -175,7 +195,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
@ -220,7 +240,77 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove("Press");
assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse();
}
@Test
public void testPublishedInformationRemovalAtDossierLevel() throws IOException {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
mockDictionaryCalls(0L);
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("Finished analysis");
// add dossier template published information and the cbi_author
var publishedInfoValue = "great job";
var cbiAuthorValue = "Kara Hunt";
dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue);
dictionary.get(DICTIONARY_AUTHOR).add(cbiAuthorValue);
reanlysisVersions.put(publishedInfoValue, 3L);
reanlysisVersions.put(cbiAuthorValue, 3L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
mockDictionaryCalls(2L);
request.setAnalysisNumber(3);
analyzeService.reanalyze(request);
EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst()
.orElseThrow();
var cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.SKIPPED, cbiAuthor.getState());
// remove at dossier level
var idRemoval = buildIdRemoval(publishedInformationEntry.getId());
idRemoval.setRemoveFromDictionary(true);
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
request.setManualRedactions(manualRedactions);
deleted.add(publishedInfoValue);
reanlysisVersions.put(publishedInfoValue, 4L);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(3L);
dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue);
request.setAnalysisNumber(3);
mockDictionaryCalls(3L);
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst()
.orElseThrow();
cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.APPLIED, cbiAuthor.getState());
// cleanup changes
deleted.remove(publishedInfoValue);
assertThat(deleted.contains(publishedInfoValue)).isFalse();
dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
assertThat(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).size()).isEqualTo(0);
dictionary.get(DICTIONARY_AUTHOR).remove(cbiAuthorValue);
assertThat(dictionary.get(DICTIONARY_AUTHOR).contains(cbiAuthorValue)).isFalse();
dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove(publishedInfoValue);
assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains(publishedInfoValue)).isFalse();
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -119,7 +119,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)