Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* implement ManualRedactionEntry Handling
This commit is contained in:
Kilian Schuettler 2023-05-03 00:06:47 +02:00
parent 8ec3c32a85
commit 40307b7e62
4 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
View File

@ -1,14 +1,21 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import java.util.Collection;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import lombok.RequiredArgsConstructor;
@ -46,4 +53,39 @@ public class ManualRedactionApplicationService {
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
}
public void applyManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) {
EntityType entityType = parseEntityType(manualRedactionEntry.getReason());
documentGraph.getPages()
.stream()
.filter(page -> isInManualRedactionEntry(manualRedactionEntry, page))
.map(PageNode::getEntities)
.flatMap(Collection::stream)
.filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId()))
.filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue()))
.map(RedactionEntity::getBoundary)
.map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph))
.peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis()))
.forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason()));
}
private EntityType parseEntityType(String reason) {
return switch (reason.toLowerCase()) {
case "false positive" -> EntityType.FALSE_POSITIVE;
case "false recommendation" -> EntityType.FALSE_RECOMMENDATION;
case "recommendation" -> EntityType.RECOMMENDATION;
default -> EntityType.ENTITY;
};
}
private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) {
return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber());
}
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -173,7 +173,6 @@ public class AnalyzeService {
} else {
nerEntities = Collections.emptyList();
}
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary,

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -63,7 +65,7 @@ class SectionFinder {
sectionsToReanalyse.add(mainNode.getTocId().get(0));
}
});
log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
return sectionsToReanalyse;
@ -80,7 +82,11 @@ class SectionFinder {
Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
Stream.concat(manualRedactions.getEntriesToAdd()
.stream()
.map(ManualRedactionEntry::getSourceId)
.filter(sourceId -> !Objects.isNull(sourceId)),
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet());
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@ -34,6 +35,7 @@ public class EntityRedactionService {
DroolsExecutionService droolsExecutionService;
EntityCreationService entityCreationService;
ManualRedactionApplicationService manualRedactionApplicationService;
public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
@ -72,6 +74,12 @@ public class EntityRedactionService {
sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node));
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
if (analyzeRequest.getManualRedactions() != null) {
analyzeRequest.getManualRedactions()
.getEntriesToAdd()
.forEach(manualRedactionEntry -> manualRedactionApplicationService.applyManualRedactionEntries(manualRedactionEntry, documentGraph));
}
log.debug("Starting Drools execution");
long ruleExecutionStart = System.currentTimeMillis();
List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,