Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* implement ManualRedactionEntry Handling
This commit is contained in:
Kilian Schuettler 2023-05-03 00:06:47 +02:00
parent 3f5e6dae66
commit 746f789be1
3 changed files with 113 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -151,37 +151,21 @@ public class AnalyzeService {
new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest);
if (sectionsToReanalyseIds.isEmpty()) {
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
}
List<SemanticNode> sectionsToReAnalyse = documentGraph.streamChildren()
.filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0)))
.collect(Collectors.toList());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode));
List<EntityRecognitionEntity> nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
} else {
nerEntities = Collections.emptyList();
}
List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary,
documentGraph,
sectionsToReAnalyse,
kieContainer,
analyzeRequest,
nerEntities);
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
entityRedactionService.applyManualRedactionEntries(analyzeRequest.getManualRedactions(), documentGraph);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
@ -198,11 +182,21 @@ public class AnalyzeService {
}
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
long startTime = System.currentTimeMillis();
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
}
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
}
private List<EntityRecognitionEntity> getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
List<EntityRecognitionEntity> nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
@ -210,13 +204,28 @@ public class AnalyzeService {
} else {
nerEntities = Collections.emptyList();
}
return nerEntities;
}
@Timed("redactmanager_analyze")
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -63,7 +65,7 @@ class SectionFinder {
sectionsToReanalyse.add(mainNode.getTocId().get(0));
}
});
log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
return sectionsToReanalyse;
@ -80,7 +82,11 @@ class SectionFinder {
Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
Stream.concat(manualRedactions.getEntriesToAdd()
.stream()
.map(ManualRedactionEntry::getSourceId)
.filter(sourceId -> !Objects.isNull(sourceId)),
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet());
}

87
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -1,6 +1,8 @@
package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@ -9,13 +11,18 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@ -34,18 +41,14 @@ public class EntityRedactionService {
DroolsExecutionService droolsExecutionService;
EntityCreationService entityCreationService;
ManualRedactionApplicationService manualRedactionApplicationService;
public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
addDictionaryEntities(dictionary, documentGraph);
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Drools Execution");
long droolsStart = System.currentTimeMillis();
@ -60,17 +63,12 @@ public class EntityRedactionService {
}
public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
List<SemanticNode> sectionsToReanalyze,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node));
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
List<SemanticNode> sectionsToReanalyze,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Drools execution");
long ruleExecutionStart = System.currentTimeMillis();
@ -87,6 +85,53 @@ public class EntityRedactionService {
}
public void applyManualRedactionEntries(ManualRedactions manualRedactions, DocumentGraph documentGraph) {
if (manualRedactions == null) {
return;
}
manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> Objects.nonNull(manualRedactionEntry.getSourceId()))
.forEach(manualRedactionEntry -> applyManualRedactionEntry(manualRedactionEntry, documentGraph));
}
private void applyManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) {
EntityType entityType = parseEntityType(manualRedactionEntry.getReason());
documentGraph.getPages()
.stream()
.filter(page -> isInManualRedactionEntry(manualRedactionEntry, page))
.map(PageNode::getEntities)
.flatMap(Collection::stream)
.filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId()))
.filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue()))
.map(RedactionEntity::getBoundary)
.map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph))
.peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis()))
.forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason()));
}
private EntityType parseEntityType(String reason) {
return switch (reason.toLowerCase()) {
case "false positive" -> EntityType.FALSE_POSITIVE;
case "false recommendation" -> EntityType.FALSE_RECOMMENDATION;
case "recommendation" -> EntityType.RECOMMENDATION;
default -> EntityType.ENTITY;
};
}
private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) {
return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber());
}
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) {