RED-6009: Document Tree Structure

* added a test for removing entities by ManualResizeRedaction in ReAnalysis
* Fixed Set logic when removing and readding a resized entity
* Refactored RedactionLog change computation

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -112,11 +112,11 @@ public class RedactionEntity {
 
         intersectingNodes.forEach(node -> node.getEntities().remove(this));
         pages.forEach(page -> page.getEntities().remove(this));
-
         intersectingNodes = new LinkedList<>();
         deepestFullyContainingNode = null;
         pages = new HashSet<>();
         removed = true;
+        ignored = true;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java

@@ -36,11 +36,13 @@ public class ManualRedactionApplicationService {
             newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
         }
 
-        entityToBeResized.setResized(true);
-        entityToBeResized.getBoundary().setStart(newStartOffset);
-        entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
         SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getTableOfContents().getRoot().getNode();
         entityToBeResized.removeFromGraph();
+        entityToBeResized.setResized(true);
+        entityToBeResized.setRemoved(false);
+        entityToBeResized.setIgnored(false);
+        entityToBeResized.getBoundary().setStart(newStartOffset);
+        entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
         entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -1,5 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -11,12 +22,6 @@ import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import org.springframework.stereotype.Service;
-
-import java.time.OffsetDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -39,57 +44,37 @@ public class RedactionChangeLogService {
             return new RedactionLogChanges(currentRedactionLog, false);
         }
 
-        List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
+        List<RedactionLogEntry> previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
-                .stream()
-                .filter(entry -> !entry.lastChangeIsRemoved())
-                .collect(Collectors.toList());
 
-        Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry()
+        Map<String, RedactionLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries);
-                .stream()
+        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries);
-                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
-                .collect(Collectors.toSet());
-
-        notRemovedPreviousEntries.forEach(added::remove);
-
-        Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
-        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
-
-        Map<String, RedactionLogEntry> addedIds = new HashMap<>();
-        added.forEach(entry -> {
-            addedIds.put(entry.getId(), entry);
-        });
-
-        Set<String> removedIds = new HashSet<>();
-        removed.forEach(entry -> {
-            removedIds.add(entry.getId());
-        });
 
         List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
 
         List<RedactionLogEntry> toRemove = new ArrayList<>();
         newRedactionLogEntries.forEach(entry -> {
-            if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
+            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
                 List<Change> changes = entry.getChanges();
                 changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
-                var newEntry = addedIds.get(entry.getId());
+                var newEntry = addedEntryIds.get(entry.getId());
                 newEntry.setChanges(changes);
-                addedIds.put(entry.getId(), newEntry);
+                addedEntryIds.put(entry.getId(), newEntry);
                 toRemove.add(entry);
             } else if (removedIds.contains(entry.getId())) {
                 entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-            } else if (addedIds.containsKey(entry.getId())) {
+            } else if (addedEntryIds.containsKey(entry.getId())) {
                 List<Change> changes = entry.getChanges();
                 changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-                var newEntry = addedIds.get(entry.getId());
+                var newEntry = addedEntryIds.get(entry.getId());
                 newEntry.setChanges(changes);
-                addedIds.put(entry.getId(), newEntry);
+                addedEntryIds.put(entry.getId(), newEntry);
                 toRemove.add(entry);
             }
         });
 
         newRedactionLogEntries.removeAll(toRemove);
 
-        addedIds.forEach((k, v) -> {
+        addedEntryIds.forEach((k, v) -> {
             if (v.getChanges().isEmpty()) {
                 v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
             }
@@ -99,7 +84,34 @@ public class RedactionChangeLogService {
         currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
 
         log.info("Change computation took: {}", System.currentTimeMillis() - start);
-        return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty());
+        return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
+    }
+
+
+    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List<RedactionLogEntry> previouslyExistingEntries) {
+
+        Set<RedactionLogEntry> removed = new HashSet<>(previouslyExistingEntries);
+        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
+
+        Set<String> removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet());
+        return removedIds;
+    }
+
+
+    private static Map<String, RedactionLogEntry> getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog,
+                                                                                                           List<RedactionLogEntry> previouslyExistingEntries) {
+
+        Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .collect(Collectors.toSet());
+
+        previouslyExistingEntries.forEach(currentExistingEntries::remove);
+        Map<String, RedactionLogEntry> addedIds = new HashMap<>();
+        currentExistingEntries.forEach(entry -> {
+            addedIds.put(entry.getId(), entry);
+        });
+        return addedIds;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -82,7 +82,7 @@ public class RedactionLogCreatorService {
         Set<String> referenceIds = new HashSet<>();
         entity.getReferences()
                 .stream()
-                .filter(redactionEntity -> !redactionEntity.isRemoved())
+                .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored())
                 .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTocId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTocId().get(0);
 
@@ -100,8 +100,8 @@ public class RedactionLogCreatorService {
                 .sectionNumber(sectionNumber)
                 .matchedRule(entity.getMatchedRule())
                 .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter().toString())
+                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore().toString())
+                .textBefore(entity.getTextBefore())
                 .startOffset(entity.getBoundary().start())
                 .endOffset(entity.getBoundary().end())
                 .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -3,7 +3,9 @@ package com.iqser.red.service.redaction.v1.server;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
+import static org.wildfly.common.Assert.assertTrue;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
@@ -25,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -55,14 +58,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -77,6 +86,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules.drl");
 
+    @Autowired
+    private EntityCreationService entityCreationService;
+
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
     @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
@@ -668,6 +680,62 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testManualResizeRedactionRemovesContainedEntities() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated.pdf");
+        File tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile();
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+        try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Desiree et al")));
+        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Melanie et al.")));
+
+        DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
+        String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
+        RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, documentGraph).findFirst().get();
+
+        String idToResize = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals("Desiree et al")).findFirst().get().getId();
+        List<Rectangle> resizedPositions = expandedEntity.getRedactionPositionsPerPage()
+                .get(0)
+                .getRectanglePerLine()
+                .stream()
+                .map(rectangle2D -> RectangleMapper.toAnnotationRectangle(rectangle2D, 3))
+                .toList();
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(idToResize)
+                .value(expandedEntityKeyword)
+                .positions(resizedPositions)
+                .status(AnnotationStatus.APPROVED)
+                .build();
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.getResizeRedactions().add(manualResizeRedaction);
+        request.setManualRedactions(manualRedactions);
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
+        tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile();
+        annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+        try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Desiree et al")));
+        assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Melanie et al.") && !entry.lastChangeIsRemoved()));
+        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals(expandedEntityKeyword)));
+    }
+
+
     @Test
     public void testTableRedactionWithCvTableService() throws IOException {
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -168,7 +168,11 @@ rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
     then
         $table.streamTableCellsWithHeader("Author(s)")
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .forEach(redactionEntity -> insert(redactionEntity));
+            .forEach(redactionEntity -> {
+                redactionEntity.addMatchedRule(6);
+                redactionEntity.setRedactionReason("Author(s) header found");
+                insert(redactionEntity);
+            });
     end
 
 rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
@@ -527,11 +531,11 @@ rule "102: Guidelines FileAttributes"
 rule "Apply manual resize redaction"
     salience 128
     when
-        $resizeRedactions: ManualResizeRedaction($id: annotationId)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId)
         $entityToBeResized: RedactionEntity(matchesAnnotationId($id))
     then
-        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions);
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
-        retract($resizeRedactions);
+        retract($resizeRedaction);
         update($entityToBeResized);
     end
 
@@ -592,6 +596,16 @@ rule "merge intersecting Entities of same type"
         insert(mergedEntity);
     end
 
+rule "remove Entity contained by Entity of same type"
+    salience 64
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
 rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when