From 5c4c8218f51f70f8b8a500d13a3f8d0d30b35786 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 28 Apr 2023 12:38:24 +0200 Subject: [PATCH] RED-6009: Document Tree Structure * added a test for removing entities by ManualResizeRedaction in ReAnalysis * Fixed Set logic when removing and readding a resized entity * Refactored RedactionLog change computation --- .../graph/entity/RedactionEntity.java | 2 +- .../ManualRedactionApplicationService.java | 8 +- .../service/RedactionChangeLogService.java | 86 +++++++++++-------- .../service/RedactionLogCreatorService.java | 6 +- .../v1/server/RedactionIntegrationTest.java | 68 +++++++++++++++ .../src/test/resources/drools/rules.drl | 22 ++++- 6 files changed, 144 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index 449b75b9..db5131ab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -112,11 +112,11 @@ public class RedactionEntity { intersectingNodes.forEach(node -> node.getEntities().remove(this)); pages.forEach(page -> page.getEntities().remove(this)); - intersectingNodes = new LinkedList<>(); deepestFullyContainingNode = null; pages = new HashSet<>(); removed = true; + ignored = true; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java index 98034bf9..0af0ad40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java @@ -36,11 +36,13 @@ public class ManualRedactionApplicationService { newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue()); } - entityToBeResized.setResized(true); - entityToBeResized.getBoundary().setStart(newStartOffset); - entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getTableOfContents().getRoot().getNode(); entityToBeResized.removeFromGraph(); + entityToBeResized.setResized(true); + entityToBeResized.setRemoved(false); + entityToBeResized.setIgnored(false); + entityToBeResized.getBoundary().setStart(newStartOffset); + entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index c91d8daa..da78f73f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -1,5 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -11,12 +22,6 @@ import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.*; -import java.util.stream.Collectors; - @Slf4j @Service @RequiredArgsConstructor @@ -39,57 +44,37 @@ public class RedactionChangeLogService { return new RedactionLogChanges(currentRedactionLog, false); } - List notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !entry.lastChangeIsRemoved()) - .collect(Collectors.toList()); + List previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList(); - Set added = currentRedactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) - .collect(Collectors.toSet()); - - notRemovedPreviousEntries.forEach(added::remove); - - Set removed = new HashSet<>(notRemovedPreviousEntries); - currentRedactionLog.getRedactionLogEntry().forEach(removed::remove); - - Map addedIds = new HashMap<>(); - added.forEach(entry -> { - addedIds.put(entry.getId(), entry); - }); - - Set removedIds = new HashSet<>(); - removed.forEach(entry -> { - removedIds.add(entry.getId()); - }); + Map addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries); + Set removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries); List newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry(); List toRemove = new ArrayList<>(); newRedactionLogEntries.forEach(entry -> { - if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { + if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) { List changes = entry.getChanges(); changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now())); - var newEntry = addedIds.get(entry.getId()); + var newEntry = addedEntryIds.get(entry.getId()); newEntry.setChanges(changes); - addedIds.put(entry.getId(), newEntry); + addedEntryIds.put(entry.getId(), newEntry); toRemove.add(entry); } else if (removedIds.contains(entry.getId())) { entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); - } else if (addedIds.containsKey(entry.getId())) { + } else if (addedEntryIds.containsKey(entry.getId())) { List changes = entry.getChanges(); changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); - var newEntry = addedIds.get(entry.getId()); + var newEntry = addedEntryIds.get(entry.getId()); newEntry.setChanges(changes); - addedIds.put(entry.getId(), newEntry); + addedEntryIds.put(entry.getId(), newEntry); toRemove.add(entry); } }); newRedactionLogEntries.removeAll(toRemove); - addedIds.forEach((k, v) -> { + addedEntryIds.forEach((k, v) -> { if (v.getChanges().isEmpty()) { v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); } @@ -99,7 +84,34 @@ public class RedactionChangeLogService { currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries); log.info("Change computation took: {}", System.currentTimeMillis() - start); - return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty()); + return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty()); + } + + + private static Set getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List previouslyExistingEntries) { + + Set removed = new HashSet<>(previouslyExistingEntries); + currentRedactionLog.getRedactionLogEntry().forEach(removed::remove); + + Set removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet()); + return removedIds; + } + + + private static Map getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog, + List previouslyExistingEntries) { + + Set currentExistingEntries = currentRedactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) + .collect(Collectors.toSet()); + + previouslyExistingEntries.forEach(currentExistingEntries::remove); + Map addedIds = new HashMap<>(); + currentExistingEntries.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); + return addedIds; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index ac8511bb..20b43a66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -82,7 +82,7 @@ public class RedactionLogCreatorService { Set referenceIds = new HashSet<>(); entity.getReferences() .stream() - .filter(redactionEntity -> !redactionEntity.isRemoved()) + .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored()) .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId()))); int sectionNumber = entity.getDeepestFullyContainingNode().getTocId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTocId().get(0); @@ -100,8 +100,8 @@ public class RedactionLogCreatorService { .sectionNumber(sectionNumber) .matchedRule(entity.getMatchedRule()) .isDictionaryEntry(entity.isDictionaryEntry()) - .textAfter(entity.getTextAfter().toString()) - .textBefore(entity.getTextBefore().toString()) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) .startOffset(entity.getBoundary().start()) .endOffset(entity.getBoundary().end()) .isDossierDictionaryEntry(entity.isDossierDictionaryEntry()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 1041b33a..0e47bf70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -3,7 +3,9 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; +import static org.wildfly.common.Assert.assertTrue; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -25,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -55,14 +58,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -77,6 +86,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules.drl"); + @Autowired + private EntityCreationService entityCreationService; + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) @@ -668,6 +680,62 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } + @Test + @SneakyThrows + public void testManualResizeRedactionRemovesContainedEntities() { + + String filePath = "files/new/crafted document.pdf"; + AnalyzeRequest request = uploadFileToStorage(filePath); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + + String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated.pdf"); + File tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile(); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Desiree et al"))); + assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Melanie et al."))); + + DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); + String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; + RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, documentGraph).findFirst().get(); + + String idToResize = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals("Desiree et al")).findFirst().get().getId(); + List resizedPositions = expandedEntity.getRedactionPositionsPerPage() + .get(0) + .getRectanglePerLine() + .stream() + .map(rectangle2D -> RectangleMapper.toAnnotationRectangle(rectangle2D, 3)) + .toList(); + ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() + .annotationId(idToResize) + .value(expandedEntityKeyword) + .positions(resizedPositions) + .status(AnnotationStatus.APPROVED) + .build(); + + ManualRedactions manualRedactions = new ManualRedactions(); + manualRedactions.getResizeRedactions().add(manualResizeRedaction); + request.setManualRedactions(manualRedactions); + AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf"); + tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile(); + annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Desiree et al"))); + assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Melanie et al.") && !entry.lastChangeIsRemoved())); + assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals(expandedEntityKeyword))); + } + + @Test public void testTableRedactionWithCvTableService() throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 0e8a51a9..9fe1798e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -168,7 +168,11 @@ rule "6.0: Add all Cell's with Header Author(s) as CBI_author" then $table.streamTableCellsWithHeader("Author(s)") .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) - .forEach(redactionEntity -> insert(redactionEntity)); + .forEach(redactionEntity -> { + redactionEntity.addMatchedRule(6); + redactionEntity.setRedactionReason("Author(s) header found"); + insert(redactionEntity); + }); end rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" @@ -527,11 +531,11 @@ rule "102: Guidelines FileAttributes" rule "Apply manual resize redaction" salience 128 when - $resizeRedactions: ManualResizeRedaction($id: annotationId) + $resizeRedaction: ManualResizeRedaction($id: annotationId) $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) then - manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions); - retract($resizeRedactions); + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); update($entityToBeResized); end @@ -592,6 +596,16 @@ rule "merge intersecting Entities of same type" insert(mergedEntity); end +rule "remove Entity contained by Entity of same type" + salience 64 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when