diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java index af83daa9..bb39eb92 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java @@ -29,8 +29,6 @@ public class TablePageBlock extends AbstractPageBlock { private String headline; private int unrotatedRowCount; private int unrotatedColCount; - private int rowCount = -1; - private int colCount = -1; private List> rows; @@ -71,19 +69,13 @@ public class TablePageBlock extends AbstractPageBlock { public int getRowCount() { - if (rowCount == -1) { - rowCount = getRows().size(); - } - return rowCount; + return getRows().size(); } public int getColCount() { - if (colCount == -1) { - colCount = getRows().stream().mapToInt(List::size).max().orElse(0); - } - return colCount; + return getRows().stream().mapToInt(List::size).max().orElse(0); } @@ -268,7 +260,7 @@ public class TablePageBlock extends AbstractPageBlock { prevX = x; } - if (prevY != null && prevX != null) { + if (prevY != null && prevX != null && !row.isEmpty()) { matrix.add(row); } prevY = y; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 228b5ab6..ba910ff7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -167,6 +167,10 @@ public class EntityCreationService { RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0)); mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason()); mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis()); + mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry)); + mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry)); + mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored)); + mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved)); addEntityToGraph(mergedEntity, node); return mergedEntity; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 7dab823d..9b005c05 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -113,11 +114,12 @@ public class RedactionLogCreatorService { public RedactionLogEntry createRedactionLogEntry(ImageNode image, String dossierTemplateId) { + String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(); return RedactionLogEntry.builder() .id(image.getId()) .color(getColor(image.getImageType().toString(), dossierTemplateId, image.isRedaction())) .isImage(true) - .type(image.getImageType().toString().toLowerCase()) + .type(imageType) .redacted(image.isRedaction()) .reason(image.getRedactionReason()) .legalBasis(image.getLegalBasis()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 8b6cbdb0..0848b794 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -131,7 +131,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } @@ -170,7 +170,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java index 62a55828..96cf4a97 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java @@ -239,7 +239,7 @@ public class PdfSegmentationServiceTest { validateTableSize(document, 1); - validateTable(document, 0, 8, 9, 0, 2); + validateTable(document, 0, 8, 8, 0, 2); List> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR", "Author, date", @@ -562,7 +562,7 @@ public class PdfSegmentationServiceTest { private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) { - Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); + TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); List> rows = table.getRows(); int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size(); @@ -576,7 +576,7 @@ public class PdfSegmentationServiceTest { private void validateTable(Document document, int tableIndex, List> values) { - Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); + TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex); List> rows = table.getRows(); List rowsFlattened = rows.stream().flatMap(List::stream).toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl deleted file mode 100644 index b0d3720c..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl +++ /dev/null @@ -1,8 +0,0 @@ -import java.util.List -import com.iqser.red.service.redaction.v1.server.DroolsListBugTest - -rule "bugtest" - when - then - DroolsListBugTest.callInFunction(); - end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 126ee375..aa27dea2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -582,6 +582,16 @@ rule "Apply image recategorization" // --------------------------------------- merging rules ------------------------------------------------------------------- +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + rule "merge intersecting Entities of same type" salience 64 when @@ -596,16 +606,6 @@ rule "merge intersecting Entities of same type" insert(mergedEntity); end -rule "remove Entity contained by Entity of same type" - salience 64 - when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) - then - $contained.removeFromGraph(); - retract($contained); - end - rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 0a638d91..893c2d1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -90,6 +90,16 @@ rule "Always redact PII" // --------------------------------------- merging rules ------------------------------------------------------------------- +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + rule "merge intersecting Entities of same type" salience 64 when @@ -163,7 +173,6 @@ rule "run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - Set redactionEntities = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); - System.out.printf("local dictionary search found %d entities\n", redactionEntities.size()); - redactionEntities.forEach(redactionEntity -> insert(redactionEntity)); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) + .forEach(redactionEntity -> insert(redactionEntity)); end