From 4aac5fb71ac6c9c0521919954f253be8cfa17f5b Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Wed, 7 Aug 2024 12:26:00 +0200 Subject: [PATCH] RED-9782: Automated Analysis should be disabled when uploading a document that... --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../v1/server/model/PrecursorEntity.java | 18 ++++++++++----- .../v1/server/service/AnalyzeService.java | 22 +++++++++++-------- .../EntityFromPrecursorCreationService.java | 13 ++++++----- .../storage/RedactionStorageService.java | 18 ++++++++++++++- 6 files changed, 53 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 1b078cad..18b95727 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.503.0" +val persistenceServiceVersion = "2.504.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index bb3fd992..8cc2a4a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.503.0" +val persistenceServiceVersion = "2.504.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index 51aaf812..a3ec0376 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -120,7 +120,8 @@ public class PrecursorEntity implements IEntity { EntityType entityType = getEntityType(entryType); String value = Optional.ofNullable(importedRedaction.getValue()) .orElse(""); - return PrecursorEntity.builder() + + PrecursorEntityBuilder precursorEntityBuilder = PrecursorEntity.builder() .id(importedRedaction.getId()) .value(value) .entityPosition(rectangleWithPages) @@ -130,14 +131,21 @@ public class PrecursorEntity implements IEntity { .orElse("")) .type(Optional.ofNullable(importedRedaction.getType()) .orElse(IMPORTED_REDACTION_TYPE)) - .section(importedRedaction.getManualOverwriteSection()) + .section(Optional.ofNullable(importedRedaction.getSection()) + .orElse("")) .entityType(entityType) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) + .manualOverwrite(new ManualChangeOverwrite(entityType)) .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA)) - .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection())) - .engines(Set.of(Engine.IMPORTED)) - .build(); + .engines(Set.of(Engine.IMPORTED)); + + if (importedRedaction.getManualOverwriteSection() != null && !importedRedaction.getManualOverwriteSection().isEmpty()) { + precursorEntityBuilder.section(importedRedaction.getManualOverwriteSection()) + .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection())); + } + + return precursorEntityBuilder.build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 5c459852..a1971a25 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -23,7 +23,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; @@ -59,6 +62,7 @@ public class AnalyzeService { RedactionServiceSettings redactionServiceSettings; NotFoundImportedEntitiesService notFoundImportedEntitiesService; FunctionTimerValues redactmanagerAnalyzePagewiseValues; + ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE; AnalysisPreparationService analysisPreparationService; @@ -198,7 +202,6 @@ public class AnalyzeService { context); } - @Timed("redactmanager_analyzeImportedRedactionsOnly") @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly") public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) { @@ -218,22 +221,23 @@ public class AnalyzeService { .flatMap(Collection::stream) .collect(Collectors.toList()); + ImportedLegalBases importedLegalBases = redactionStorageService.getImportedLegalBases(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded Imported Legal Bases for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, analysisData.document(), notFoundManualOrImportedEntries, analysisData.dictionary().getVersion(), analysisData.kieWrapperEntityRules().rulesVersion()); + entityLogChanges.getEntityLog() + .setLegalBasis(importedLegalBases.getImportedLegalBases() + .stream() + .map(importedLegalBasisMapper::toEntityLogLegalBasis) + .toList()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); - return finalizeAnalysis(analyzeRequest, - startTime, - analysisData.kieWrapperComponentRules(), - entityLogChanges, - analysisData.document(), - analysisData.document().getNumberOfPages(), - false, - new HashSet<>(), + return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), entityLogChanges, analysisData.document(), analysisData.document().getNumberOfPages(), false, new HashSet<>(), context); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java index 8df13405..6977a630 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; @@ -127,11 +128,13 @@ public class EntityFromPrecursorCreationService { precursorEntity.getEntityType(), closestEntity.getDeepestFullyContainingNode()); } else { - correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), - precursorEntity.type(), - precursorEntity.getEntityType(), - precursorEntity.getId(), - precursorEntity.getManualOverwrite().getSection().orElse(null)); + String section = precursorEntity.getManualOverwrite().getSection() + .orElse(null); + if (section != null && precursorEntity.getEngines().contains(Engine.IMPORTED) && section.isBlank() && !precursorEntity.getSection().isBlank()) { + section = precursorEntity.getSection(); + } + + correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId(), section); } correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 24da363c..93b816f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -15,7 +15,9 @@ import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -23,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -152,6 +153,21 @@ public class RedactionStorageService { } + + @Timed("redactmanager_getImportedLegalBases") + public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES), + ImportedLegalBases.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("Imported legal bases not available."); + return new ImportedLegalBases(); + } + } + + @Deprecated(forRemoval = true) @Timed("redactmanager_getRedactionLog") public RedactionLog getRedactionLog(String dossierId, String fileId) {