From 92fc003576b9e2f1ccb59702bc6887b5ef6dbb5b Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Wed, 7 Aug 2024 12:26:02 +0200 Subject: [PATCH] RED-9782: Automated Analysis should be disabled when uploading a document that... --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../v1/server/model/PrecursorEntity.java | 18 ++++++++---- .../v1/server/service/AnalyzeService.java | 29 +++++++++---------- .../EntityFromPrecursorCreationService.java | 13 +++++---- .../storage/RedactionStorageService.java | 17 +++++++++++ 6 files changed, 53 insertions(+), 28 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index f29df24a..ad778a39 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.465.38" +val persistenceServiceVersion = "2.465.41" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 504cc4bc..9e4d88b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.142.6" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.465.38" +val persistenceServiceVersion = "2.465.41" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index 51aaf812..a3ec0376 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -120,7 +120,8 @@ public class PrecursorEntity implements IEntity { EntityType entityType = getEntityType(entryType); String value = Optional.ofNullable(importedRedaction.getValue()) .orElse(""); - return PrecursorEntity.builder() + + PrecursorEntityBuilder precursorEntityBuilder = PrecursorEntity.builder() .id(importedRedaction.getId()) .value(value) .entityPosition(rectangleWithPages) @@ -130,14 +131,21 @@ public class PrecursorEntity implements IEntity { .orElse("")) .type(Optional.ofNullable(importedRedaction.getType()) .orElse(IMPORTED_REDACTION_TYPE)) - .section(importedRedaction.getManualOverwriteSection()) + .section(Optional.ofNullable(importedRedaction.getSection()) + .orElse("")) .entityType(entityType) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) + .manualOverwrite(new ManualChangeOverwrite(entityType)) .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA)) - .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection())) - .engines(Set.of(Engine.IMPORTED)) - .build(); + .engines(Set.of(Engine.IMPORTED)); + + if (importedRedaction.getManualOverwriteSection() != null && !importedRedaction.getManualOverwriteSection().isEmpty()) { + precursorEntityBuilder.section(importedRedaction.getManualOverwriteSection()) + .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection())); + } + + return precursorEntityBuilder.build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 31b4ca14..c47a72cb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -23,8 +23,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; @@ -76,6 +78,7 @@ public class AnalyzeService { ImportedRedactionEntryService importedRedactionEntryService; ObservedStorageService observedStorageService; FunctionTimerValues redactmanagerAnalyzePagewiseValues; + ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE; @Timed("redactmanager_reanalyze") @@ -245,10 +248,9 @@ public class AnalyzeService { } - @Timed("redactmanager_analyzeImportedRedactionsOnly") @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly") - public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest){ + public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) { long startTime = System.currentTimeMillis(); @@ -260,30 +262,25 @@ public class AnalyzeService { ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + ImportedLegalBases importedLegalBases = redactionStorageService.getImportedLegalBases(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded Imported Legal Bases for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); + EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, document, notFoundImportedEntries, new DictionaryVersion(0, 0), 0); - EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, - document, - notFoundImportedEntries, - new DictionaryVersion(0,0), - 0); + entityLogChanges.getEntityLog() + .setLegalBasis(importedLegalBases.getImportedLegalBases() + .stream() + .map(importedLegalBasisMapper::toEntityLogLegalBasis) + .toList()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries); - return finalizeAnalysis(analyzeRequest, - startTime, - KieWrapper.empty(), - entityLogChanges, - document, - document.getNumberOfPages(), - false, - new HashSet<>()); + return finalizeAnalysis(analyzeRequest, startTime, KieWrapper.empty(), entityLogChanges, document, document.getNumberOfPages(), false, new HashSet<>()); } - private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java index 8df13405..6977a630 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; @@ -127,11 +128,13 @@ public class EntityFromPrecursorCreationService { precursorEntity.getEntityType(), closestEntity.getDeepestFullyContainingNode()); } else { - correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), - precursorEntity.type(), - precursorEntity.getEntityType(), - precursorEntity.getId(), - precursorEntity.getManualOverwrite().getSection().orElse(null)); + String section = precursorEntity.getManualOverwrite().getSection() + .orElse(null); + if (section != null && precursorEntity.getEngines().contains(Engine.IMPORTED) && section.isBlank() && !precursorEntity.getSection().isBlank()) { + section = precursorEntity.getSection(); + } + + correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId(), section); } correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 6f641017..dfbe2bb8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -9,10 +9,12 @@ import java.util.Set; import java.util.stream.Collectors; import org.springframework.cache.annotation.Cacheable; +import org.springframework.context.annotation.Import; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -147,6 +149,21 @@ public class RedactionStorageService { } + + @Timed("redactmanager_getImportedLegalBases") + public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES), + ImportedLegalBases.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("Imported legal bases not available."); + return new ImportedLegalBases(); + } + } + + @Deprecated(forRemoval = true) @Timed("redactmanager_getRedactionLog") public RedactionLog getRedactionLog(String dossierId, String fileId) {