Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

WIP: Draft: draft #325

Closed
andrei.isvoran.ext wants to merge 1 commits from RED-8776 into master
pull from: RED-8776
merge into: Marmelator:master
Conversation 3 Commits 1 Files Changed 3 +47 -10
3 changed files with 47 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -22,16 +22,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -40,6 +33,8 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
@ -79,6 +74,7 @@ public class AnalyzeService {
ImportedRedactionEntryService importedRedactionEntryService;
ObservedStorageService observedStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
EntityEnrichmentService entityEnrichmentService;
@Timed("redactmanager_reanalyze")
@ -148,6 +144,9 @@ public class AnalyzeService {
dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
entityCreationService.createTextEntitiesForRecategorizedDictionaryEntries(analyzeRequest, previousEntityLog.getEntityLogEntry(), document);
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
@ -215,6 +214,10 @@ public class AnalyzeService {
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var previousEntityLogEntries = entityLogCreatorService.getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
entityCreationService.createTextEntitiesForRecategorizedDictionaryEntries(analyzeRequest, previousEntityLogEntries, document);
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -83,7 +83,7 @@ public class EntityLogCreatorService {
}
private List<EntityLogEntry> getPreviousEntityLogEntries(String dossierId, String fileId) {
public List<EntityLogEntry> getPreviousEntityLogEntries(String dossierId, String fileId) {
if (redactionStorageService.entityLogExists(dossierId, fileId)) {
return redactionStorageService.getEntityLog(dossierId, fileId).getEntityLogEntry();

38
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -1,6 +1,12 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Collection;
@ -18,7 +24,10 @@ import org.apache.commons.lang3.tuple.Pair;
import org.kie.api.runtime.KieSession;
import com.google.common.base.Functions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
@ -621,7 +630,32 @@ public class EntityCreationService {
}
// Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
public void createTextEntitiesForRecategorizedDictionaryEntries(AnalyzeRequest analyzeRequest, List<EntityLogEntry> previousEntityLogEntries, SemanticNode node) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
previousEntityLogEntries.stream()
.filter(entityLogEntry -> !entityLogEntry.getManualChanges().isEmpty())
.filter(entityLogEntry -> entityLogEntry.getManualChanges()
.stream()
.anyMatch(manualChange -> manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE)))
.filter(entityLogEntry -> entityLogEntry.getManualChanges()
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType() != ManualRedactionType.REMOVE
|| entityLogEntry.getManualChanges()
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType() != ManualRedactionType.REMOVE_FROM_DICTIONARY)
.forEach(entityLogEntry -> {
TextRange textRange = new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset());
TextEntity textEntity = TextEntity.initialEntityNode(textRange, entityLogEntry.getType(), EntityType.ENTITY, entityLogEntry.getId());
var manualRedactionsForThisEntity = analyzeRequest.getManualRedactions().buildAll()
.stream()
.filter(manualRedaction -> manualRedaction.getAnnotationId().equals(entityLogEntry.getId()))
.toList();
textEntity.getManualOverwrite().addChanges(manualRedactionsForThisEntity);
textEntity.addEngines(entityLogEntry.getEngines());
entityCreationService.addEntityToGraph(textEntity, node);
});
}
// Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
@Deprecated(forRemoval = true)
public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {