From 1800a89b399598b3a85e7900221b7449a5791a3d Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Fri, 12 Jul 2024 10:30:58 +0200 Subject: [PATCH] RED-9246: Resize on dossier level for dossier-template-level entry becomes skipped --- .../server/model/dictionary/Dictionary.java | 3 + .../resources/drools/all_rules_documine.drl | 5 +- .../AbstractRedactionIntegrationTest.java | 127 +++++-------- .../redaction/v1/server/AnalysisTest.java | 17 +- .../v1/server/DocumineFloraTest.java | 14 +- .../v1/server/MigrationIntegrationTest.java | 4 +- .../v1/server/RedactionAcceptanceTest.java | 10 +- .../v1/server/RedactionIntegrationTest.java | 172 +++++++++++++++--- .../v1/server/RedactionIntegrationV2Test.java | 17 +- .../redaction/v1/server/RulesTest.java | 52 +++--- .../DocumentPerformanceIntegrationTest.java | 13 +- .../ManualChangesEnd2EndTest.java | 19 +- .../UnprocessedChangesServiceTest.java | 16 +- .../resources/drools/acceptance_rules.drl | 6 +- .../drools/all_redact_manager_rules.drl | 9 +- .../test/resources/drools/documine_flora.drl | 6 +- .../resources/drools/efsa_sanitisation.drl | 6 +- .../drools/manual_redaction_rules.drl | 2 +- .../src/test/resources/drools/rules.drl | 9 +- .../src/test/resources/drools/rules_v2.drl | 6 +- .../src/test/resources/drools/table_demo.drl | 6 +- .../src/test/resources/drools/test_rules.drl | 6 +- .../resources/all_redact_manager_rules.drl | 9 +- .../src/main/resources/all_rules_documine.drl | 5 +- 24 files changed, 293 insertions(+), 246 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 472a610f..a650d374 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -31,6 +31,9 @@ public class Dictionary { @Getter private List dictionaryModels; + // todo: dossier and dossier template level DictionaryModels override each other + // at the moment there are no problems because they always have the same rank / hint information + // but it should be changed so that the localAccessMap contains all models private Map localAccessMap = new HashMap<>(); @Getter diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index ffd89bca..0498e7b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -1455,11 +1455,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -1598,6 +1598,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 8154d748..fb412bd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -105,8 +105,9 @@ public abstract class AbstractRedactionIntegrationTest { protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; protected final static String TEST_DOSSIER_ID = "123"; protected final static String TEST_FILE_ID = "123"; - public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; - public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; + public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; + public static final String DOSSIER_AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID; public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String FORMULA_TYPE_ID = FORMULA_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String SIGNATURE_TYPE_ID = SIGNATURE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -124,7 +125,6 @@ public abstract class AbstractRedactionIntegrationTest { public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID; - public static final String DOSSIER_AUTHOR_TYPE_ID = AUTHOR_TYPE_ID + ":" + TEST_DOSSIER_ID; public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -192,7 +192,8 @@ public abstract class AbstractRedactionIntegrationTest { protected final Map rankTypeMap = new HashMap<>(); protected final Colors colors = new Colors(); protected final Map reanlysisVersions = new HashMap<>(); - protected final Set deleted = new HashSet<>(); + protected final Set deletedInTemplate = new HashSet<>(); + protected final Set deletedInDossier = new HashSet<>(); @MockBean protected RulesClient rulesClient; @@ -267,8 +268,8 @@ public abstract class AbstractRedactionIntegrationTest { true)); when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true)); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, version, true)).then((Answer>)invocation -> (getTypeResponseForTemplate())); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, version, true)).then((Answer>)invocation -> ((getTypeResponseForDossier()))); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, version, true)).then((Answer>)invocation -> (getTemplateDictionaryTypeResponse())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, version, true)).then((Answer>)invocation -> ((getDossierDictionaryTypeResponse()))); } @@ -421,6 +422,7 @@ public abstract class AbstractRedactionIntegrationTest { typeColorMap.put(FORMULA_INDICATOR, "#ffe187"); typeColorMap.put(SIGNATURE_INDICATOR, "#ffe187"); typeColorMap.put(IMPORTED_REDACTION_INDICATOR, "#fcfbe6"); + typeColorMap.put(DOSSIER_REDACTIONS_INDICATOR, "#ffe187"); typeColorMap.put(ROTATE_SIMPLE_INDICATOR, "#66ccff"); hintTypeMap.put(VERTEBRATE_INDICATOR, true); @@ -523,86 +525,56 @@ public abstract class AbstractRedactionIntegrationTest { } - protected List getTypeResponseForTemplate() { + protected List getTemplateDictionaryTypeResponse() { - return typeColorMap.entrySet() + return dictionary.keySet() .stream() - .map(typeColor -> Type.builder() - .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(typeColor.getKey()) + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(key) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor(typeColor.getValue()) - .isHint(hintTypeMap.get(typeColor.getKey())) - .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) - .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) - .rank(rankTypeMap.get(typeColor.getKey())) - .entries(toDictionaryEntry(dictionary.get(typeColor.getKey()))) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) + .entries(toDictionaryEntry(dictionary.get(key), false)) + .build()) + .collect(Collectors.toList()); + } + + + + protected List getDossierDictionaryTypeResponse() { + + return dossierDictionary.keySet() + .stream() + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID) + .type(key) + .dossierId(TEST_DOSSIER_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) + .entries(toDictionaryEntry(dossierDictionary.get(key), true)) .build()) .toList(); } - protected List getTypeResponseForDossier() { - - return List.of(Type.builder() - .id(IMPORTED_REDACTION_TYPE_ID) - .type(IMPORTED_REDACTION_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .dossierId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(IMPORTED_REDACTION_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(IMPORTED_REDACTION_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(IMPORTED_REDACTION_INDICATOR)) - .rank(rankTypeMap.get(IMPORTED_REDACTION_INDICATOR)) - .entries(toDictionaryEntry(dossierDictionary.get(IMPORTED_REDACTION_INDICATOR))) - .build(), - Type.builder() - .id(DOSSIER_REDACTIONS_TYPE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .dossierId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .entries(toDictionaryEntry(dossierDictionary.get(DOSSIER_REDACTIONS_INDICATOR))) - .build(), - Type.builder() - .id(DOSSIER_AUTHOR_TYPE_ID) - .type(DICTIONARY_AUTHOR) - .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .dossierId(TEST_DOSSIER_ID) - .hexColor("#ffe184") - .isHint(hintTypeMap.get(DICTIONARY_AUTHOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR)) - .isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR)) - .rank(rankTypeMap.get(DICTIONARY_AUTHOR)) - .entries(toDictionaryEntry(dossierDictionary.get(DICTIONARY_AUTHOR))) - .build(), - Type.builder() - .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID) - .type(PUBLISHED_INFORMATION_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .dossierId(TEST_DOSSIER_ID) - .hexColor("#ffe180") - .isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) - .rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR)) - .entries(toDictionaryEntry(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR))) - .build()); - } - - protected Type getDictionaryResponse(String type, boolean isDossierDictionary) { + String id = type + ":" + TEST_DOSSIER_TEMPLATE_ID; + return Type.builder() - .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID) + .id(isDossierDictionary ? id + ":" + TEST_DOSSIER_ID : id) .hexColor(typeColorMap.get(type)) - .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) - .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>()) - .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>()) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type), true) : toDictionaryEntry(dictionary.get(type), false)) + .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type), isDossierDictionary) : new ArrayList<>()) + .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type), isDossierDictionary) : new ArrayList<>()) .isHint(hintTypeMap.get(type)) .isCaseInsensitive(caseInSensitiveMap.get(type)) .isRecommendation(recommendationTypeMap.get(type)) @@ -617,20 +589,21 @@ public abstract class AbstractRedactionIntegrationTest { } - private List toDictionaryEntry(List entries) { + private List toDictionaryEntry(List entries, boolean isDossierDictionary) { if (entries == null) { return Collections.emptyList(); } return entries.stream() - .map(this::toDictionaryEntry) + .map(entry -> toDictionaryEntry(entry, isDossierDictionary)) .collect(Collectors.toList()); } - private DictionaryEntry toDictionaryEntry(String entry) { + private DictionaryEntry toDictionaryEntry(String entry, boolean isDossierDictionary) { + var deleted = isDossierDictionary ? deletedInDossier : deletedInTemplate; return DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java index ea4f8acd..138935b1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java @@ -101,23 +101,10 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR - + ":" - + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 30c34ff1..d96438fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -217,22 +217,14 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES)); + loadDictionaryForTest(); loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java index 24a8f956..01774794 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java @@ -84,10 +84,10 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier()); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 1f60f43d..c7dad2bc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -99,10 +99,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier()); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); @@ -273,7 +273,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build(); request.setManualRedactions(manualRedactions); - deleted.add(publishedInfoValue); + deletedInDossier.add(publishedInfoValue); reanlysisVersions.put(publishedInfoValue, 4L); when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(3L); @@ -293,8 +293,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { assertEquals(EntryState.APPLIED, cbiAuthor.getState()); // cleanup changes - deleted.remove(publishedInfoValue); - assertThat(deleted.contains(publishedInfoValue)).isFalse(); + deletedInTemplate.remove(publishedInfoValue); + assertThat(deletedInTemplate.contains(publishedInfoValue)).isFalse(); dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); assertThat(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).size()).isEqualTo(0); dictionary.get(DICTIONARY_AUTHOR).remove(cbiAuthorValue); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index ec004448..d2eb85de 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -27,7 +27,10 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -85,6 +88,7 @@ import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedactionIntegrationTest extends RulesIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules.drl"); @@ -118,9 +122,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier()); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); @@ -492,8 +497,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); - deleted.add("David Chubb"); - deleted.add("mouse"); + deletedInTemplate.add("David Chubb"); + deletedInTemplate.add("mouse"); reanlysisVersions.put("mouse", 3L); @@ -528,7 +533,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - deleted.remove("mouse"); + deletedInTemplate.remove("mouse"); reanlysisVersions.put("mouse", 4L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L); @@ -544,6 +549,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { @Test + @Order(2) public void testChangeComputation() throws IOException { String fileName = "files/syngenta/CustomerFiles/SinglePages/test1S1T1.pdf"; @@ -554,15 +560,15 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - dictionary.get(DICTIONARY_AUTHOR).add("report"); - reanlysisVersions.put("report", 2L); + dictionary.get(DICTIONARY_AUTHOR).add("EFSA conclusion"); + reanlysisVersions.put("EFSA conclusion", 2L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L); mockDictionaryCalls(0L); analyzeService.reanalyze(request); - dictionary.get(DICTIONARY_AUTHOR).add("assessment report"); - reanlysisVersions.put("assessment report", 3L); + dictionary.get(DICTIONARY_AUTHOR).add("draft EFSA conclusion"); + reanlysisVersions.put("draft EFSA conclusion", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); mockDictionaryCalls(2L); analyzeService.reanalyze(request); @@ -575,7 +581,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { var changes = entityLog.getEntityLogEntry() .stream() - .filter(entry -> entry.getValue() != null && entry.getValue().equals("report")) + .filter(entry -> entry.getValue() != null && entry.getValue().equals("EFSA conclusion")) .findFirst() .get().getChanges(); @@ -648,8 +654,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { dictionary.get(DICTIONARY_AUTHOR).add("physical"); reanlysisVersions.put("physical", 2L); - deleted.add("David Chubb"); - deleted.add("mouse"); + deletedInTemplate.add("David Chubb"); + deletedInTemplate.add("mouse"); reanlysisVersions.put("David Chubb", 3L); @@ -686,7 +692,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - deleted.remove("mouse"); + deletedInTemplate.remove("mouse"); reanlysisVersions.put("mouse", 4L); mockDictionaryCalls(3L); @@ -2107,6 +2113,90 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { } + @Test + @SneakyThrows + @Order(1) + public void testShrinkTemplateEntryAndAddToDossierLevel() { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + String pdfFile = "files/syngenta/CustomerFiles/SinglePages/test1S1T1.pdf"; + + String manualAddId = UUID.randomUUID().toString(); + + List fullPositions = List.of(Rectangle.builder().topLeftX(190.85f).topLeftY(332.5033f).width(228.0134f).height(13.645125f).page(1).build()); + + List shrunkPositions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(333.45f).width(71.40744f).height(12.823441f).page(1).build()); + ManualRedactionEntry manualRedactionEntry = getFullManualRedactionEntry(manualAddId, fullPositions); + manualRedactionEntry.setAddToDictionary(true); + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + Set entriesToAdd = Set.of(manualRedactionEntry); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).build()); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + request.setAnalysisNumber(1); + + String commentsOnTheAssessmentReport = "Comments on the assessment report"; + dictionary.get(DICTIONARY_AUTHOR).add(commentsOnTheAssessmentReport); + reanlysisVersions.put(commentsOnTheAssessmentReport, 0L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + mockDictionaryCalls(0L); + + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertTrue(entityLog.getEntityLogEntry() + .stream() + .findFirst() + .isPresent()); + EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getValue().equals(commentsOnTheAssessmentReport)) + .findFirst() + .get(); + String id = entityLogEntry1.getId(); + + assertEquals(entityLogEntry1.getState(), EntryState.APPLIED); + + manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); + Set resizeRedactions = Set.of(getManualResizeRedaction(id, shrunkPositions)); + + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).resizeRedactions(resizeRedactions).build()); + request.setAnalysisNumber(2); + + dossierDictionary.get(DICTIONARY_AUTHOR).add(commentsOnTheAssessmentReport); + deletedInDossier.add(commentsOnTheAssessmentReport); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); + String assessment = "assessment"; + dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment); + reanlysisVersions.put(assessment, 1L); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(id))); + entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(id)) + .findFirst() + .get(); + var entityLogEntry2 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getValue().equals(assessment)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.APPLIED); + assertEquals(entityLogEntry2.getState(), EntryState.APPLIED); + assertThat(entityLogEntry2.getStartOffset() > entityLogEntry1.getEndOffset()); + } + + private IdRemoval getIdRemoval(String id) { return IdRemoval.builder() @@ -2122,18 +2212,52 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { private ManualRedactionEntry getManualRedactionEntry(String id, List positions, String reason) { - ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry(); - manualRedactionEntry2.setAnnotationId(id); - manualRedactionEntry2.setFileId("fileId"); - manualRedactionEntry2.setUser("test"); - manualRedactionEntry2.setType("manual"); - manualRedactionEntry2.setRectangle(false); - manualRedactionEntry2.setRequestDate(OffsetDateTime.now()); - manualRedactionEntry2.setValue("assessment"); - manualRedactionEntry2.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); - manualRedactionEntry2.setReason(reason); - manualRedactionEntry2.setPositions(positions); - return manualRedactionEntry2; + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(id); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("manual"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue("assessment"); + manualRedactionEntry.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + manualRedactionEntry.setReason(reason); + manualRedactionEntry.setPositions(positions); + return manualRedactionEntry; + } + + + private ManualRedactionEntry getFullManualRedactionEntry(String id, List positions) { + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(id); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("manual"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue("Comments on the assessment report"); + manualRedactionEntry.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + manualRedactionEntry.setReason( + "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); + manualRedactionEntry.setPositions(positions); + return manualRedactionEntry; + } + + + private ManualResizeRedaction getManualResizeRedaction(String id, List positions) { + + ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction(); + manualResizeRedaction.setAnnotationId(id); + manualResizeRedaction.setFileId("fileId"); + manualResizeRedaction.setUser("test"); + manualResizeRedaction.setRequestDate(OffsetDateTime.now()); + manualResizeRedaction.setValue("assessment"); + manualResizeRedaction.setValue("assessment"); + manualResizeRedaction.setPositions(positions); + manualResizeRedaction.setAddToAllDossiers(false); + manualResizeRedaction.setUpdateDictionary(true); + return manualResizeRedaction; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 83a1bac1..ef8a2147 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -75,23 +75,10 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR - + ":" - + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 56ce9aaf..ea5fc214 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -279,19 +279,10 @@ public class RulesTest { loadDictionaryForTest(); loadTypeForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, anyLong(), false)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, anyLong(), false)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); mockDictionaryCalls(0L); @@ -784,21 +775,40 @@ public class RulesTest { } - private List getTypeResponse() { + protected List getTemplateDictionaryTypeResponse() { - return typeColorMap.entrySet() + return dictionary.keySet() .stream() - .map(typeColor -> Type.builder() - .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(typeColor.getKey()) + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(key) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .hexColor(typeColor.getValue()) - .isHint(hintTypeMap.get(typeColor.getKey())) - .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) - .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) - .rank(rankTypeMap.get(typeColor.getKey())) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) .build()) + .collect(Collectors.toList()); + } + + + protected List getDossierDictionaryTypeResponse() { + + return dossierDictionary.keySet() + .stream() + .map(key -> Type.builder() + .id(key + ":" + TEST_DOSSIER_TEMPLATE_ID + ":" + TEST_DOSSIER_ID) + .type(key) + .dossierId(TEST_DOSSIER_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(key)) + .isHint(hintTypeMap.get(key)) + .isCaseInsensitive(caseInSensitiveMap.get(key)) + .isRecommendation(recommendationTypeMap.get(key)) + .rank(rankTypeMap.get(key)) + .build()) .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 020e7cf0..4ac29688 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -110,19 +110,10 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 26e0eab7..627289f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -123,24 +123,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate()); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTemplateDictionaryTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR - + ":" - + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index d996f346..6295723a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -99,22 +99,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT loadTypeForTest(); loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get( - DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); - + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 48bd6e54..0ab463d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -1245,11 +1244,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -1401,6 +1400,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 7cc86d57..5d9ac5c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -733,7 +732,8 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), - anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"), + !removed() ) then entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -1898,11 +1898,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -2054,6 +2054,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 6f92b3d1..1eff5d4a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -55,7 +55,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -1310,11 +1309,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -1453,6 +1452,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 886fa4cc..5b5c14d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -804,11 +803,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -960,6 +959,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 34955669..b42602ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -301,6 +300,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 8530db58..6616c138 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -504,7 +503,8 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), - anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"), + !removed() ) then entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -1333,11 +1333,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -1489,6 +1489,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 6a89aefa..0c9b0b67 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -256,11 +255,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -412,6 +411,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index a25e4590..aca5480f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -380,11 +379,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -547,6 +546,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index c722d8da..1c6dcd8f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -63,7 +63,6 @@ global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary global RulesLogger logger - //------------------------------------ queries ------------------------------------ query "getFileAttributes" @@ -280,11 +279,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -447,6 +446,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index d460093f..6e447a8e 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -732,7 +732,8 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), - anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"), + !removed() ) then entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -1893,11 +1894,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -2048,12 +2049,12 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); end - //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index ffd89bca..0498e7b5 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1455,11 +1455,11 @@ rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when $larger: TextEntity($type: type(), $entityType: entityType, !removed()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) then $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + update($contained); end @@ -1598,6 +1598,7 @@ rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dict when $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");