From a598022c2b1afdedd347a511017d6a6e570e1531 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Mon, 18 Mar 2024 10:47:04 +0200 Subject: [PATCH] draft --- .../v1/server/service/AnalyzeService.java | 17 +++++---- .../service/EntityLogCreatorService.java | 2 +- .../document/EntityCreationService.java | 38 ++++++++++++++++++- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 6fa7fb31..15efc494 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -22,16 +22,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; -import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; -import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; @@ -40,6 +33,8 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; @@ -79,6 +74,7 @@ public class AnalyzeService { ImportedRedactionEntryService importedRedactionEntryService; ObservedStorageService observedStorageService; FunctionTimerValues redactmanagerAnalyzePagewiseValues; + EntityEnrichmentService entityEnrichmentService; @Timed("redactmanager_reanalyze") @@ -148,6 +144,9 @@ public class AnalyzeService { dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService.createTextEntitiesForRecategorizedDictionaryEntries(analyzeRequest, previousEntityLog.getEntityLogEntry(), document); + // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), document, @@ -215,6 +214,10 @@ public class AnalyzeService { dictionarySearchService.addDictionaryEntities(dictionary, document); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + var previousEntityLogEntries = entityLogCreatorService.getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService.createTextEntitiesForRecategorizedDictionaryEntries(analyzeRequest, previousEntityLogEntries, document); + // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), document, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ecedefc5..26754ce1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -83,7 +83,7 @@ public class EntityLogCreatorService { } - private List getPreviousEntityLogEntries(String dossierId, String fileId) { + public List getPreviousEntityLogEntries(String dossierId, String fileId) { if (redactionStorageService.entityLogExists(dossierId, fileId)) { return redactionStorageService.getEntityLog(dossierId, fileId).getEntityLogEntry(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index f8b803d2..c0c69983 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1,6 +1,12 @@ package com.iqser.red.service.redaction.v1.server.service.document; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.Collection; @@ -18,7 +24,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.kie.api.runtime.KieSession; import com.google.common.base.Functions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; @@ -621,7 +630,32 @@ public class EntityCreationService { } - // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here. + public void createTextEntitiesForRecategorizedDictionaryEntries(AnalyzeRequest analyzeRequest, List previousEntityLogEntries, SemanticNode node) { + + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + previousEntityLogEntries.stream() + .filter(entityLogEntry -> !entityLogEntry.getManualChanges().isEmpty()) + .filter(entityLogEntry -> entityLogEntry.getManualChanges() + .stream() + .anyMatch(manualChange -> manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) + .filter(entityLogEntry -> entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType() != ManualRedactionType.REMOVE + || entityLogEntry.getManualChanges() + .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType() != ManualRedactionType.REMOVE_FROM_DICTIONARY) + .forEach(entityLogEntry -> { + TextRange textRange = new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset()); + TextEntity textEntity = TextEntity.initialEntityNode(textRange, entityLogEntry.getType(), EntityType.ENTITY, entityLogEntry.getId()); + var manualRedactionsForThisEntity = analyzeRequest.getManualRedactions().buildAll() + .stream() + .filter(manualRedaction -> manualRedaction.getAnnotationId().equals(entityLogEntry.getId())) + .toList(); + textEntity.getManualOverwrite().addChanges(manualRedactionsForThisEntity); + textEntity.addEngines(entityLogEntry.getEngines()); + entityCreationService.addEntityToGraph(textEntity, node); + }); + } + + // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here. @Deprecated(forRemoval = true) public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) { -- 2.47.2