diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 228b5ab6..ba910ff7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -167,6 +167,10 @@ public class EntityCreationService { RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0)); mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason()); mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis()); + mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry)); + mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry)); + mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored)); + mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved)); addEntityToGraph(mergedEntity, node); return mergedEntity; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 8b6cbdb0..0848b794 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -131,7 +131,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } @@ -170,7 +170,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java index 62a55828..ee80e6ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java @@ -562,7 +562,7 @@ public class PdfSegmentationServiceTest { private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) { - Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); + TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); List> rows = table.getRows(); int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size(); @@ -576,7 +576,7 @@ public class PdfSegmentationServiceTest { private void validateTable(Document document, int tableIndex, List> values) { - Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); + TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); List> rows = table.getRows(); List rowsFlattened = rows.stream().flatMap(List::stream).toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl deleted file mode 100644 index b0d3720c..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl +++ /dev/null @@ -1,8 +0,0 @@ -import java.util.List -import com.iqser.red.service.redaction.v1.server.DroolsListBugTest - -rule "bugtest" - when - then - DroolsListBugTest.callInFunction(); - end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 126ee375..aa27dea2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -582,6 +582,16 @@ rule "Apply image recategorization" // --------------------------------------- merging rules ------------------------------------------------------------------- +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + rule "merge intersecting Entities of same type" salience 64 when @@ -596,16 +606,6 @@ rule "merge intersecting Entities of same type" insert(mergedEntity); end -rule "remove Entity contained by Entity of same type" - salience 64 - when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) - then - $contained.removeFromGraph(); - retract($contained); - end - rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 0a638d91..893c2d1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -90,6 +90,16 @@ rule "Always redact PII" // --------------------------------------- merging rules ------------------------------------------------------------------- +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + rule "merge intersecting Entities of same type" salience 64 when @@ -163,7 +173,6 @@ rule "run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - Set redactionEntities = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); - System.out.printf("local dictionary search found %d entities\n", redactionEntities.size()); - redactionEntities.forEach(redactionEntity -> insert(redactionEntity)); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) + .forEach(redactionEntity -> insert(redactionEntity)); end