diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index b49ef282..0ecfdbd6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.96.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.377.0" +val persistenceServiceVersion = "2.379.0" val springBootStarterVersion = "3.1.5" configurations { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 8b928ecd..d8bc0065 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -166,6 +166,7 @@ public final class MigrationEntity { case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; case RECOMMENDATION -> EntryType.RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; + default -> EntryType.FALSE_POSITIVE; }; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java index 1214d6d3..4f780146 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.io.Serializable; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; @@ -11,11 +10,11 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import lombok.AllArgsConstructor; import lombok.Data; +import lombok.extern.slf4j.Slf4j; @Data -@AllArgsConstructor +@Slf4j public class DictionaryModel implements Serializable { private final String type; @@ -29,6 +28,7 @@ public class DictionaryModel implements Serializable { private final Set falseRecommendations; private transient SearchImplementation entriesSearch; + private transient SearchImplementation deletionEntriesSearch; private transient SearchImplementation falsePositiveSearch; private transient SearchImplementation falseRecommendationsSearch; @@ -56,20 +56,6 @@ public class DictionaryModel implements Serializable { this.entries = entries; this.falsePositives = falsePositives; this.falseRecommendations = falseRecommendations; - - this.entriesSearch = new SearchImplementation(this.entries.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntryModel::getValue) - .collect(Collectors.toList()), caseInsensitive); - this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntryModel::getValue) - .collect(Collectors.toList()), caseInsensitive); - this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); - } @@ -94,6 +80,18 @@ public class DictionaryModel implements Serializable { } + public SearchImplementation getDeletionEntriesSearch() { + + if (deletionEntriesSearch == null) { + this.deletionEntriesSearch = new SearchImplementation(this.entries.stream() + .filter(DictionaryEntry::isDeleted) + .map(DictionaryEntry::getValue) + .collect(Collectors.toList()), caseInsensitive); + } + return deletionEntriesSearch; + } + + public SearchImplementation getFalsePositiveSearch() { if (falsePositiveSearch == null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java index 6fb2d781..1550265e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java @@ -5,5 +5,6 @@ public enum EntityType { HINT, RECOMMENDATION, FALSE_POSITIVE, - FALSE_RECOMMENDATION + FALSE_RECOMMENDATION, + DICTIONARY_REMOVAL } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index a0880c90..ae6f23ce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -40,7 +40,7 @@ public class TextEntity implements IEntity { TextRange textRange; @Builder.Default List duplicateTextRanges = new ArrayList<>(); - String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted + String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted final EntityType entityType; @Builder.Default @@ -182,25 +182,37 @@ public class TextEntity implements IEntity { public boolean containedBy(TextEntity textEntity) { - return this.textRange.containedBy(textEntity.getTextRange()) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::containedBy)); + return this.textRange.containedBy(textEntity.getTextRange()) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges() + .stream() + .anyMatch(duplicateTextRange::containedBy)); } public boolean contains(TextEntity textEntity) { - return this.textRange.contains(textEntity.getTextRange()) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::contains)); + return this.textRange.contains(textEntity.getTextRange()) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges() + .stream() + .anyMatch(duplicateTextRange::contains)); } public boolean intersects(TextEntity textEntity) { - return this.textRange.intersects(textEntity.getTextRange()) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) - || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::intersects)); + return this.textRange.intersects(textEntity.getTextRange()) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) // + || duplicateTextRanges.stream() + .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges() + .stream() + .anyMatch(duplicateTextRange::intersects)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 136b26f1..661afdf7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -6,10 +6,11 @@ import java.util.Set; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; @@ -38,10 +39,13 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - for (var model : dictionary.getDictionaryModels()) { + for (DictionaryModel model : dictionary.getDictionaryModels()) { bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); + if (model.isDossierDictionary()) { + bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); + } } } @@ -52,11 +56,12 @@ public class DictionarySearchService { SemanticNode node, boolean isDossierDictionaryEntry) { + Set engines = Set.of(Engine.DICTIONARY); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) .stream() .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) - .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)) + .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines) .ifPresent(entity -> { entity.setDictionaryEntry(true); entity.setDossierDictionaryEntry(isDossierDictionaryEntry); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index e91e1d26..fda9874b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -1,9 +1,9 @@ package com.iqser.red.service.redaction.v1.server.service; import java.awt.Color; -import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -368,30 +368,23 @@ public class DictionaryService { @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary") public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { - List mergedDictionaries; + List mergedDictionaries = new LinkedList<>(); - var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); - var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary(); + DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); + List dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary(); + dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); - // merge dictionaries if they have same names - long dossierDictionaryVersion = -1; - if (dossierDictionaryExists(dossierId)) { - var dossierRepresentation = getDossierDictionary(dossierId); - var dossierDictionaries = dossierRepresentation.getDictionary(); - mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), - convertDictionaryModel(dossierDictionaries))); - dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); - } else { - mergedDictionaries = new ArrayList<>(); - dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); - } + // add dossier + DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId); + List dossierDictionaries = dossierRepresentation.getDictionary(); + dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); return new Dictionary(mergedDictionaries.stream() .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) .collect(Collectors.toList()), DictionaryVersion.builder() .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()) - .dossierVersion(dossierDictionaryVersion) + .dossierVersion(dossierRepresentation.getDictionaryVersion()) .build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ecedefc5..040f432d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -52,9 +52,11 @@ public class EntityLogCreatorService { RedactionStorageService redactionStorageService; - private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) { + private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) { - return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)); + return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) // + || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION) // + || textEntity.getEntityType().equals(EntityType.DICTIONARY_REMOVAL)); } @@ -145,7 +147,7 @@ public class EntityLogCreatorService { document.getEntities() .stream() .filter(entity -> !entity.getValue().isEmpty()) - .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) + .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) .filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); document.streamAllImages() @@ -341,6 +343,7 @@ public class EntityLogCreatorService { case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; case RECOMMENDATION -> EntryType.RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; + case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE; }; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 1fdcbc73..9be7e540 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -233,18 +233,13 @@ public class DictionaryServiceTest { dictionaryService.updateDictionary("dtId", "dossierId"); var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); - assertThat(dict.getDictionaryModels().size()).isEqualTo(1); + assertThat(dict.getDictionaryModels().size()).isEqualTo(2); var dictModel = dict.getDictionaryModels() .get(0); assertThat(dictModel.getType()).isEqualTo(type); - assertThat(dictModel.getEntries().size()).isEqualTo(4); + assertThat(dictModel.getEntries().size()).isEqualTo(3); dictModel.getEntries() - .forEach(entry -> { - switch (entry.getValue()) { - case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId()); - case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()); - } - }); + .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId())); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index cf23ee96..2b0f63d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -1335,6 +1335,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index a8c81adb..3a064872 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -1037,6 +1037,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end + // Rule unit: PII.4 rule "PII.4.0: Redact line after contact information keywords" when @@ -1340,8 +1341,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix" .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end - -// Rule unit: PII.12 rule "PII.12.1: Expand PII entities with salutation prefix" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -1351,6 +1350,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end + // Rule unit: PII.13 rule "PII.13.0: Add recommendation for PII after Contact Person" when @@ -1542,6 +1542,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end + // Rule unit: ETC.9 rule "ETC.9.0: Redact skipped impurities" when @@ -1559,6 +1560,7 @@ rule "ETC.9.1: Redact impurities" $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end + // Rule unit: ETC.10 rule "ETC.10.0: Redact Product Composition Information" when @@ -1567,6 +1569,7 @@ rule "ETC.10.0: Redact Product Composition Information" $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009"); end + // Rule unit: ETC.11 rule "ETC.11.0: Recommend first line in table cell with name and address of owner" when @@ -1596,6 +1599,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end + //------------------------------------ AI rules ------------------------------------ // Rule unit: AI.0 @@ -1641,7 +1645,8 @@ rule "AI.3.0: Recommend authors from AI as PII" .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); end -//------------------------------------ Manual redaction rules ------------------------------------ + +//------------------------------------ Manual changes rules ------------------------------------ // Rule unit: MAN.0 rule "MAN.0.0: Apply manual resize redaction" @@ -1761,7 +1766,6 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end - rule "MAN.3.3: Apply recategorization entities by default" salience 128 when @@ -1770,6 +1774,7 @@ rule "MAN.3.3: Apply recategorization entities by default" $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 @@ -1930,6 +1935,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end + // Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when @@ -1940,6 +1946,7 @@ rule "X.9.0: Merge mostly contained signatures" $signature.addEngine(LayoutEngine.AI); end + // Rule unit: X.10 rule "X.10.0: remove false positives of ai" when @@ -1949,6 +1956,21 @@ rule "X.10.0: remove false positives of ai" $aiSignature.remove("X.10.0", "Removed because false positive"); end + +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 5daeb74e..f6a02f4d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1437,6 +1437,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 4648f756..a2837b89 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -296,6 +296,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC }); end + // Rule unit: CBI.23 rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when @@ -315,6 +316,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end + //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 @@ -504,6 +506,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end + // Rule unit: PII.10 rule "PII.10.0: Redact study director abbreviation" when @@ -947,6 +950,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index cd1da993..c53bd38a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -278,6 +278,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ Local dictionary search rules ------------------------------------ // Rule unit: LDS.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 4afae8d3..86c36dab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -274,7 +274,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end - rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when @@ -602,7 +601,6 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve //------------------------------------ PII rules ------------------------------------ - // Rule unit: PII.0 rule "PII.0.0: Redact all PII" when @@ -1425,6 +1423,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 6126142e..25fb3092 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -402,6 +402,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index cd219ac8..30ae324b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -523,6 +523,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 3208beea..11a37645 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -439,6 +439,20 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java index 7d1aa705..e8bb5be2 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java @@ -33,11 +33,7 @@ public class RuleFileMigrator { rulesToAdd.forEach(ruleFileBluePrint::addRule); } - RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(), - combinedBluePrint.globals(), - combinedBluePrint.queries(), - ruleFileBluePrint.ruleClasses()); - String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java index d14aa1a0..ffddc766 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java @@ -17,6 +17,7 @@ public class RuleType { "ETC", "Other",// "MAN", "Manual changes",// "X", "Entity merging",// + "DICT", "Dictionary merging",// "FA", "File attributes",// "LDS", "Local dictionary search",// "TAB", "Table extraction",// diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index a8c81adb..4ddb246c 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -1949,6 +1949,21 @@ rule "X.10.0: remove false positives of ai" $aiSignature.remove("X.10.0", "Removed because false positive"); end + +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 214730fe..03650c43 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1582,6 +1582,19 @@ rule "X.10.0: remove false positives of ai" end +//------------------------------------ Dictionary merging rules ------------------------------------ + +// Rule unit: DICT.0 +rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL" + salience 64 + when + $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active()) + $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); + end + //------------------------------------ File attributes rules ------------------------------------ diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt index fcc625f2..bc919f53 100644 --- a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt +++ b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt @@ -8,5 +8,6 @@ X.5.0 X.5.1 X.6.* X.8.* +DICT.*.* FA.*.* LDS.*.* \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt index 2cce8ab4..0808a08b 100644 --- a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt +++ b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt @@ -7,5 +7,6 @@ X.5.0 X.5.1 X.7.0 X.8.* +DICT.*.* FA.*.* LDS.*.* \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/rules-management/src/main/resources/order_template.txt index e56b9bff..d46eef6c 100644 --- a/redaction-service-v1/rules-management/src/main/resources/order_template.txt +++ b/redaction-service-v1/rules-management/src/main/resources/order_template.txt @@ -9,5 +9,6 @@ ETC AI MAN X +DICT FA LDS \ No newline at end of file