Merge branch 'hotfix' into 'master'

hotfix: filter entities with empty value in redactionLog and entityLog

See merge request redactmanager/redaction-service!215

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -113,7 +113,8 @@ public class AnalyzeService {
                     entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
                             dictionaryIncrement.getDictionaryVersion(),
                             analyzeRequest.getDossierTemplateId(),
-                            false), document,
+                            false),
+                    document,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
@@ -209,7 +210,11 @@ public class AnalyzeService {
                 dictionary.getVersion(),
                 kieWrapperEntityRules.rulesVersion());
 
-        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
+        return finalizeAnalysis(analyzeRequest,
+                startTime,
+                kieWrapperComponentRules,
+                new EntityLogChanges(entityLog, false),
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionary.getVersion(),
@@ -226,7 +231,8 @@ public class AnalyzeService {
                                                     Set<Integer> sectionsToReanalyseIds) {
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+                analyzeRequest.getDossierTemplateId(),
+                notFoundManualRedactionEntries);
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -236,12 +242,18 @@ public class AnalyzeService {
 
         previousRedactionLog.getRedactionLogEntry()
                 .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
+
         previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+
         return previousRedactionLog;
     }
 
 
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
+                                           long startTime,
+                                           KieWrapper kieWrapperComponentRules,
+                                           EntityLogChanges entityLogChanges,
+                                           Document document,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
@@ -284,7 +296,9 @@ public class AnalyzeService {
     }
 
 
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                      KieWrapper kieWrapperComponentRules,
+                                                      Document document,
                                                       Set<FileAttribute> addedFileAttributes,
                                                       EntityLogChanges entityLogChanges,
                                                       DictionaryVersion dictionaryVersion) {
@@ -293,7 +307,9 @@ public class AnalyzeService {
             return;
         }
 
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                entityLogChanges.getEntityLog(),
+                document,
                 addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -377,7 +393,8 @@ public class AnalyzeService {
                                             KieWrapper wrapper) {
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+                analyzeRequest.getDossierTemplateId(),
+                notFoundManualRedactionEntries);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -121,6 +121,7 @@ public class EntityLogCreatorService {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
         Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> !entry.getType()
@@ -150,6 +151,7 @@ public class EntityLogCreatorService {
         Set<String> processedIds = new HashSet<>();
         document.getEntities()
                 .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -47,8 +47,12 @@ public class RedactionLogCreatorService {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
         Set<String> processIds = new HashSet<>();
-        document.getEntities().stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
+        document.getEntities()
-                .filter(IEntity::active).forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
+                .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
+                .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(IEntity::active)
+                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
         document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
         notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
@@ -95,7 +99,8 @@ public class RedactionLogCreatorService {
         entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
         boolean isHint = isHint(entity.getEntityType());
-        return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
+        return RedactionLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@@ -185,7 +190,9 @@ public class RedactionLogCreatorService {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder().id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
+        return RedactionLogEntry.builder()
+                .id(image.getId())
+                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
                 .isImage(true)
                 .value(image.value())
                 .type(imageType)
@@ -235,8 +242,14 @@ public class RedactionLogCreatorService {
                 .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
                 .textAfter("")
                 .textBefore("")
-                .startOffset(-1).endOffset(-1).positions(manualEntity.getManualOverwrite().getPositions().orElse(manualEntity.getEntityPosition())
+                .startOffset(-1)
-                        .stream().map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
+                .endOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
                 .engines(Collections.emptySet())
                 .reference(Collections.emptySet())
                 .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.storage;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.stream.Collectors;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
@@ -90,7 +92,11 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
+                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
+                    RedactionLog.class);
+            redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return redactionLog;
         } catch (StorageObjectDoesNotExist e) {
             log.debug("RedactionLog not available.");
             return null;
@@ -98,13 +104,16 @@ public class RedactionStorageService {
 
     }
 
+
     @Timed("redactmanager_getRedactionLog")
     public EntityLog getEntityLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            entityLog.setEntityLogEntry(entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return entityLog;
         } catch (StorageObjectDoesNotExist e) {
-            log.debug("RedactionLog not available.");
+            log.debug("EntityLog not available.");
             return null;
         }
 
@@ -116,10 +125,18 @@ public class RedactionStorageService {
 
         try {
             return DocumentData.builder()
-                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), DocumentStructure.class))
+                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
-                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), DocumentTextData[].class))
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
-                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), DocumentPositionData[].class))
+                            DocumentStructure.class))
-                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), DocumentPage[].class))
+                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
+                            DocumentTextData[].class))
+                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
+                            DocumentPositionData[].class))
+                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
+                            DocumentPage[].class))
                     .build();
         } catch (StorageObjectDoesNotExist e) {
             log.debug("DocumentData not available.");