RED-6009: Document Tree Structure
* added a test for removing entities by ManualResizeRedaction in ReAnalysis * Fixed Set logic when removing and readding a resized entity * Refactored RedactionLog change computation
This commit is contained in:
Kilian Schuettler
parent
94e5203d8a
commit
808e0f1e86
@ -112,11 +112,11 @@ public class RedactionEntity {
|
||||
|
||||
intersectingNodes.forEach(node -> node.getEntities().remove(this));
|
||||
pages.forEach(page -> page.getEntities().remove(this));
|
||||
|
||||
intersectingNodes = new LinkedList<>();
|
||||
deepestFullyContainingNode = null;
|
||||
pages = new HashSet<>();
|
||||
removed = true;
|
||||
ignored = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -36,11 +36,13 @@ public class ManualRedactionApplicationService {
|
||||
newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
|
||||
}
|
||||
|
||||
entityToBeResized.setResized(true);
|
||||
entityToBeResized.getBoundary().setStart(newStartOffset);
|
||||
entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getTableOfContents().getRoot().getNode();
|
||||
entityToBeResized.removeFromGraph();
|
||||
entityToBeResized.setResized(true);
|
||||
entityToBeResized.setRemoved(false);
|
||||
entityToBeResized.setIgnored(false);
|
||||
entityToBeResized.getBoundary().setStart(newStartOffset);
|
||||
entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
@ -11,12 +22,6 @@ import io.micrometer.core.annotation.Timed;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ -39,57 +44,37 @@ public class RedactionChangeLogService {
|
||||
return new RedactionLogChanges(currentRedactionLog, false);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> !entry.lastChangeIsRemoved())
|
||||
.collect(Collectors.toList());
|
||||
List<RedactionLogEntry> previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
|
||||
|
||||
Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
notRemovedPreviousEntries.forEach(added::remove);
|
||||
|
||||
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
|
||||
currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
|
||||
|
||||
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
|
||||
added.forEach(entry -> {
|
||||
addedIds.put(entry.getId(), entry);
|
||||
});
|
||||
|
||||
Set<String> removedIds = new HashSet<>();
|
||||
removed.forEach(entry -> {
|
||||
removedIds.add(entry.getId());
|
||||
});
|
||||
Map<String, RedactionLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries);
|
||||
Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries);
|
||||
|
||||
List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
|
||||
|
||||
List<RedactionLogEntry> toRemove = new ArrayList<>();
|
||||
newRedactionLogEntries.forEach(entry -> {
|
||||
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
|
||||
if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
var newEntry = addedEntryIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
addedEntryIds.put(entry.getId(), newEntry);
|
||||
toRemove.add(entry);
|
||||
} else if (removedIds.contains(entry.getId())) {
|
||||
entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
|
||||
} else if (addedIds.containsKey(entry.getId())) {
|
||||
} else if (addedEntryIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
var newEntry = addedEntryIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
addedEntryIds.put(entry.getId(), newEntry);
|
||||
toRemove.add(entry);
|
||||
}
|
||||
});
|
||||
|
||||
newRedactionLogEntries.removeAll(toRemove);
|
||||
|
||||
addedIds.forEach((k, v) -> {
|
||||
addedEntryIds.forEach((k, v) -> {
|
||||
if (v.getChanges().isEmpty()) {
|
||||
v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
|
||||
}
|
||||
@ -99,7 +84,34 @@ public class RedactionChangeLogService {
|
||||
currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
|
||||
|
||||
log.info("Change computation took: {}", System.currentTimeMillis() - start);
|
||||
return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty());
|
||||
return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
|
||||
}
|
||||
|
||||
|
||||
private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List<RedactionLogEntry> previouslyExistingEntries) {
|
||||
|
||||
Set<RedactionLogEntry> removed = new HashSet<>(previouslyExistingEntries);
|
||||
currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
|
||||
|
||||
Set<String> removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet());
|
||||
return removedIds;
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, RedactionLogEntry> getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog,
|
||||
List<RedactionLogEntry> previouslyExistingEntries) {
|
||||
|
||||
Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
previouslyExistingEntries.forEach(currentExistingEntries::remove);
|
||||
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
|
||||
currentExistingEntries.forEach(entry -> {
|
||||
addedIds.put(entry.getId(), entry);
|
||||
});
|
||||
return addedIds;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -82,7 +82,7 @@ public class RedactionLogCreatorService {
|
||||
Set<String> referenceIds = new HashSet<>();
|
||||
entity.getReferences()
|
||||
.stream()
|
||||
.filter(redactionEntity -> !redactionEntity.isRemoved())
|
||||
.filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored())
|
||||
.forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
|
||||
int sectionNumber = entity.getDeepestFullyContainingNode().getTocId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTocId().get(0);
|
||||
|
||||
@ -100,8 +100,8 @@ public class RedactionLogCreatorService {
|
||||
.sectionNumber(sectionNumber)
|
||||
.matchedRule(entity.getMatchedRule())
|
||||
.isDictionaryEntry(entity.isDictionaryEntry())
|
||||
.textAfter(entity.getTextAfter().toString())
|
||||
.textBefore(entity.getTextBefore().toString())
|
||||
.textAfter(entity.getTextAfter())
|
||||
.textBefore(entity.getTextBefore())
|
||||
.startOffset(entity.getBoundary().start())
|
||||
.endOffset(entity.getBoundary().end())
|
||||
.isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
||||
|
||||
@ -3,7 +3,9 @@ package com.iqser.red.service.redaction.v1.server;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.wildfly.common.Assert.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
@ -25,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
@ -55,14 +58,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
@ -77,6 +86,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
||||
|
||||
private static final String RULES = loadFromClassPath("drools/rules.drl");
|
||||
|
||||
@Autowired
|
||||
private EntityCreationService entityCreationService;
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
@ -668,6 +680,62 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testManualResizeRedactionRemovesContainedEntities() {
|
||||
|
||||
String filePath = "files/new/crafted document.pdf";
|
||||
AnalyzeRequest request = uploadFileToStorage(filePath);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated.pdf");
|
||||
File tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile();
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Desiree et al")));
|
||||
assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Melanie et al.")));
|
||||
|
||||
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
|
||||
String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
|
||||
RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, documentGraph).findFirst().get();
|
||||
|
||||
String idToResize = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals("Desiree et al")).findFirst().get().getId();
|
||||
List<Rectangle> resizedPositions = expandedEntity.getRedactionPositionsPerPage()
|
||||
.get(0)
|
||||
.getRectanglePerLine()
|
||||
.stream()
|
||||
.map(rectangle2D -> RectangleMapper.toAnnotationRectangle(rectangle2D, 3))
|
||||
.toList();
|
||||
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
|
||||
.annotationId(idToResize)
|
||||
.value(expandedEntityKeyword)
|
||||
.positions(resizedPositions)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.build();
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
|
||||
|
||||
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
|
||||
tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile();
|
||||
annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Desiree et al")));
|
||||
assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Melanie et al.") && !entry.lastChangeIsRemoved()));
|
||||
assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals(expandedEntityKeyword)));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTableRedactionWithCvTableService() throws IOException {
|
||||
|
||||
|
||||
@ -168,7 +168,11 @@ rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
|
||||
then
|
||||
$table.streamTableCellsWithHeader("Author(s)")
|
||||
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
||||
.forEach(redactionEntity -> insert(redactionEntity));
|
||||
.forEach(redactionEntity -> {
|
||||
redactionEntity.addMatchedRule(6);
|
||||
redactionEntity.setRedactionReason("Author(s) header found");
|
||||
insert(redactionEntity);
|
||||
});
|
||||
end
|
||||
|
||||
rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
|
||||
@ -527,11 +531,11 @@ rule "102: Guidelines FileAttributes"
|
||||
rule "Apply manual resize redaction"
|
||||
salience 128
|
||||
when
|
||||
$resizeRedactions: ManualResizeRedaction($id: annotationId)
|
||||
$resizeRedaction: ManualResizeRedaction($id: annotationId)
|
||||
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
|
||||
then
|
||||
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions);
|
||||
retract($resizeRedactions);
|
||||
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
|
||||
retract($resizeRedaction);
|
||||
update($entityToBeResized);
|
||||
end
|
||||
|
||||
@ -592,6 +596,16 @@ rule "merge intersecting Entities of same type"
|
||||
insert(mergedEntity);
|
||||
end
|
||||
|
||||
rule "remove Entity contained by Entity of same type"
|
||||
salience 64
|
||||
when
|
||||
$larger: RedactionEntity($type: type, $entityType: entityType)
|
||||
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
|
||||
then
|
||||
$contained.removeFromGraph();
|
||||
retract($contained);
|
||||
end
|
||||
|
||||
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||
salience 64
|
||||
when
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user