From 66ac1429b04a9e17fef5730a413ab73841500816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Thu, 30 Nov 2023 13:25:17 +0100 Subject: [PATCH] hotfix: filter entities with empty value in redactionLog and entityLog --- .../v1/server/service/AnalyzeService.java | 31 ++++++++++++++----- .../service/EntityLogCreatorService.java | 2 ++ .../service/RedactionLogCreatorService.java | 25 +++++++++++---- .../storage/RedactionStorageService.java | 31 ++++++++++++++----- 4 files changed, 69 insertions(+), 20 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 7dbf0dbf..2db7c1f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -113,7 +113,8 @@ public class AnalyzeService { entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, dictionaryIncrement.getDictionaryVersion(), analyzeRequest.getDossierTemplateId(), - false), document, + false), + document, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -209,7 +210,11 @@ public class AnalyzeService { dictionary.getVersion(), kieWrapperEntityRules.rulesVersion()); - return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document, + return finalizeAnalysis(analyzeRequest, + startTime, + kieWrapperComponentRules, + new EntityLogChanges(entityLog, false), + document, redactionLog, document.getNumberOfPages(), dictionary.getVersion(), @@ -226,7 +231,8 @@ public class AnalyzeService { Set sectionsToReanalyseIds) { List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + analyzeRequest.getDossierTemplateId(), + notFoundManualRedactionEntries); var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), @@ -236,12 +242,18 @@ public class AnalyzeService { previousRedactionLog.getRedactionLogEntry() .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); + previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); + return previousRedactionLog; } - private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document, + private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, + long startTime, + KieWrapper kieWrapperComponentRules, + EntityLogChanges entityLogChanges, + Document document, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, @@ -284,7 +296,9 @@ public class AnalyzeService { } - private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document, + private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, + KieWrapper kieWrapperComponentRules, + Document document, Set addedFileAttributes, EntityLogChanges entityLogChanges, DictionaryVersion dictionaryVersion) { @@ -293,7 +307,9 @@ public class AnalyzeService { return; } - List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document, + List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), + entityLogChanges.getEntityLog(), + document, addedFileAttributes.stream().toList()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -377,7 +393,8 @@ public class AnalyzeService { KieWrapper wrapper) { List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + analyzeRequest.getDossierTemplateId(), + notFoundManualRedactionEntries); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 3d8c00a2..1cfd4afe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -121,6 +121,7 @@ public class EntityLogCreatorService { List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() .stream() .filter(entry -> !entry.getType() @@ -150,6 +151,7 @@ public class EntityLogCreatorService { Set processedIds = new HashSet<>(); document.getEntities() .stream() + .filter(entity -> !entity.getValue().isEmpty()) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds))); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 8892ba63..968e16c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -47,8 +47,12 @@ public class RedactionLogCreatorService { List entries = new ArrayList<>(); Set processIds = new HashSet<>(); - document.getEntities().stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation) - .filter(IEntity::active).forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds))); + document.getEntities() + .stream() + .filter(entity -> !entity.getValue().isEmpty()) + .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation) + .filter(IEntity::active) + .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds))); document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId))); notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId))); return entries; @@ -95,7 +99,8 @@ public class RedactionLogCreatorService { entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); boolean isHint = isHint(entity.getEntityType()); - return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) + return RedactionLogEntry.builder() + .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) @@ -185,7 +190,9 @@ public class RedactionLogCreatorService { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); - return RedactionLogEntry.builder().id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) + return RedactionLogEntry.builder() + .id(image.getId()) + .color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) .isImage(true) .value(image.value()) .type(imageType) @@ -235,8 +242,14 @@ public class RedactionLogCreatorService { .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) .textAfter("") .textBefore("") - .startOffset(-1).endOffset(-1).positions(manualEntity.getManualOverwrite().getPositions().orElse(manualEntity.getEntityPosition()) - .stream().map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList()) + .startOffset(-1) + .endOffset(-1) + .positions(manualEntity.getManualOverwrite() + .getPositions() + .orElse(manualEntity.getEntityPosition()) + .stream() + .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) .engines(Collections.emptySet()) .reference(Collections.emptySet()) .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index e1107813..87682cda 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.storage; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; + import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; @@ -90,7 +92,11 @@ public class RedactionStorageService { public RedactionLog getRedactionLog(String dossierId, String fileId) { try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class); + RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), + RedactionLog.class); + redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList())); + return redactionLog; } catch (StorageObjectDoesNotExist e) { log.debug("RedactionLog not available."); return null; @@ -98,13 +104,16 @@ public class RedactionStorageService { } + @Timed("redactmanager_getRedactionLog") public EntityLog getEntityLog(String dossierId, String fileId) { try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); + EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); + entityLog.setEntityLogEntry(entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList())); + return entityLog; } catch (StorageObjectDoesNotExist e) { - log.debug("RedactionLog not available."); + log.debug("EntityLog not available."); return null; } @@ -116,10 +125,18 @@ public class RedactionStorageService { try { return DocumentData.builder() - .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), DocumentStructure.class)) - .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), DocumentTextData[].class)) - .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), DocumentPositionData[].class)) - .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), DocumentPage[].class)) + .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), + DocumentStructure.class)) + .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), + DocumentTextData[].class)) + .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), + DocumentPositionData[].class)) + .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), + DocumentPage[].class)) .build(); } catch (StorageObjectDoesNotExist e) { log.debug("DocumentData not available.");