RED-6009 - Document Tree Structure

* fix rebased tests

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java

@@ -29,8 +29,6 @@ public class TablePageBlock extends AbstractPageBlock {
     private String headline;
     private int unrotatedRowCount;
     private int unrotatedColCount;
-    private int rowCount = -1;
-    private int colCount = -1;
     private List<List<Cell>> rows;
 
 
@@ -71,19 +69,13 @@ public class TablePageBlock extends AbstractPageBlock {
 
     public int getRowCount() {
 
-        if (rowCount == -1) {
+        return getRows().size();
-            rowCount = getRows().size();
-        }
-        return rowCount;
     }
 
 
     public int getColCount() {
 
-        if (colCount == -1) {
+        return getRows().stream().mapToInt(List::size).max().orElse(0);
-            colCount = getRows().stream().mapToInt(List::size).max().orElse(0);
-        }
-        return colCount;
 
     }
 
@@ -268,7 +260,7 @@ public class TablePageBlock extends AbstractPageBlock {
                 prevX = x;
             }
 
-            if (prevY != null && prevX != null) {
+            if (prevY != null && prevX != null && !row.isEmpty()) {
                 matrix.add(row);
             }
             prevY = y;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -167,6 +167,10 @@ public class EntityCreationService {
         RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
         mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
         mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
+        mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry));
+        mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry));
+        mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored));
+        mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
 
         addEntityToGraph(mergedEntity, node);
         return mergedEntity;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -113,11 +114,12 @@ public class RedactionLogCreatorService {
 
     public RedactionLogEntry createRedactionLogEntry(ImageNode image, String dossierTemplateId) {
 
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase();
         return RedactionLogEntry.builder()
                 .id(image.getId())
                 .color(getColor(image.getImageType().toString(), dossierTemplateId, image.isRedaction()))
                 .isImage(true)
-                .type(image.getImageType().toString().toLowerCase())
+                .type(imageType)
                 .redacted(image.isRedaction())
                 .reason(image.getRedactionReason())
                 .legalBasis(image.getLegalBasis())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -131,7 +131,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
-        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
         assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
 
     }
@@ -170,7 +170,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
-        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
         assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -239,7 +239,7 @@ public class PdfSegmentationServiceTest {
 
         validateTableSize(document, 1);
 
-        validateTable(document, 0, 8, 9, 0, 2);
+        validateTable(document, 0, 8, 8, 0, 2);
 
         List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                         "Author, date",
@@ -562,7 +562,7 @@ public class PdfSegmentationServiceTest {
 
     private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
 
-        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
         int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
 
@@ -576,7 +576,7 @@ public class PdfSegmentationServiceTest {
 
     private void validateTable(Document document, int tableIndex, List<List<String>> values) {
 
-        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
 
         List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl

@@ -1,8 +0,0 @@
-import java.util.List
-import com.iqser.red.service.redaction.v1.server.DroolsListBugTest
-
-rule "bugtest"
-    when
-    then
-        DroolsListBugTest.callInFunction();
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -582,6 +582,16 @@ rule "Apply image recategorization"
 
 // --------------------------------------- merging rules -------------------------------------------------------------------
 
+rule "remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
 rule "merge intersecting Entities of same type"
     salience 64
     when
@@ -596,16 +606,6 @@ rule "merge intersecting Entities of same type"
         insert(mergedEntity);
     end
 
-rule "remove Entity contained by Entity of same type"
-    salience 64
-    when
-        $larger: RedactionEntity($type: type, $entityType: entityType)
-        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
-    then
-        $contained.removeFromGraph();
-        retract($contained);
-    end
-
 rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -90,6 +90,16 @@ rule "Always redact PII"
 
 // --------------------------------------- merging rules -------------------------------------------------------------------
 
+rule "remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
 rule "merge intersecting Entities of same type"
     salience 64
     when
@@ -163,7 +173,6 @@ rule "run local dictionary search"
     when
         DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
     then
-        Set<RedactionEntity> redactionEntities = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
+        entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
-        System.out.printf("local dictionary search found %d entities\n", redactionEntities.size());
+            .forEach(redactionEntity -> insert(redactionEntity));
-        redactionEntities.forEach(redactionEntity -> insert(redactionEntity));
     end