RED-7784 - Fixes for unprocessed redactions

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java

@@ -9,9 +9,11 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
@@ -19,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@@ -36,6 +39,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
 
 import lombok.AccessLevel;
@@ -47,6 +51,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 @Slf4j
+@Transactional
 public class EntityLogService {
 
     FileManagementStorageService fileManagementStorageService;
@@ -56,6 +61,7 @@ public class EntityLogService {
     DossierService dossierService;
     CommentService commentService;
     DictionaryPersistenceService dictionaryPersistenceService;
+    DossierTemplatePersistenceService dossierTemplatePersistenceService;
 
 
     public EntityLog getEntityLog(String dossierId, String fileId) {
@@ -84,7 +90,7 @@ public class EntityLogService {
             DossierEntity dossier = dossierService.getDossierById(dossierId);
             ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
             List<UnprocessedManualEntity> unprocessedManualEntities = getUnprocessedManualEntities(dossierId, fileId, dossier.getDossierTemplateId(), manualRedactions);
-            mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier.getDossierTemplateId());
+            mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier, fileStatus);
         }
 
         Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);
@@ -136,20 +142,20 @@ public class EntityLogService {
 
     }
 
-    public void mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, String dossierTemplateId) {
+    public void mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
 
         log.info("Merging EntityLog");
-        mergeManualRedactionEntries(manualRedactions.getEntriesToAdd(), unprocessedManualEntities, entityLog, dossierTemplateId);
+        mergeManualRedactionEntries(manualRedactions.getEntriesToAdd(), unprocessedManualEntities, entityLog, dossier, fileStatus);
         mergeIdsToRemove(manualRedactions.getIdsToRemove(), entityLog);
         mergeResizeRedactions(manualRedactions.getResizeRedactions(), entityLog);
         mergeLegalBasisChanges(manualRedactions.getLegalBasisChanges(), entityLog);
-        mergeRecategorizations(manualRedactions.getRecategorizations(), entityLog);
+        mergeRecategorizations(manualRedactions.getRecategorizations(), entityLog, dossier, fileStatus);
         mergeForceRedactions(manualRedactions.getForceRedactions(), entityLog);
         log.info("EntityLog merged successfully!");
     }
 
 
-    private void mergeManualRedactionEntries(Set<ManualRedactionEntry> manualRedactionEntries, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, String dossierTemplateId) {
+    private void mergeManualRedactionEntries(Set<ManualRedactionEntry> manualRedactionEntries, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
 
         manualRedactionEntries.forEach(manualRedactionEntry -> {
             UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream()
@@ -161,12 +167,12 @@ public class EntityLogService {
                     .dateTime(OffsetDateTime.now())
                     .type(ChangeType.ADDED)
                     .build());
-            boolean isHint = isHint(manualRedactionEntry.getType(), dossierTemplateId);
+            boolean isHint = isHint(manualRedactionEntry.getType(), dossier, fileStatus);
             entityLog.getEntityLogEntry().add(EntityLogEntry.builder()
                     .id(manualRedactionEntry.getAnnotationId())
                     .type(manualRedactionEntry.getType())
                     .value(manualRedactionEntry.getValue())
-                    .legalBasis(manualRedactionEntry.getLegalBasis())
+                    .legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis())
                     .reason(manualRedactionEntry.getReason())
                     .entryType(isHint ? EntryType.HINT : EntryType.ENTITY)
                     .state(isHint ? EntryState.SKIPPED : EntryState.APPLIED)
@@ -262,12 +268,15 @@ public class EntityLogService {
         });
     }
 
-    private void mergeRecategorizations(Set<ManualRecategorization> recategorizations, EntityLog entityLog) {
+    private void mergeRecategorizations(Set<ManualRecategorization> recategorizations, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
 
         recategorizations.forEach(recategorization -> {
+            boolean isHint = isHint(recategorization.getType(), dossier, fileStatus);
             var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
             if (entity.isPresent()) {
                 entity.get().setType(recategorization.getType());
+                entity.get().setEntryType(getEntryType(isHint, recategorization.getType()));
+                entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
                 addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
                 entity.get().getManualChanges().add(ManualChange.builder()
                         .manualRedactionType(ManualRedactionType.RECATEGORIZE)
@@ -280,6 +289,17 @@ public class EntityLogService {
         });
     }
 
+
+    private static EntryType getEntryType(boolean isHint, String type) {
+
+        if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) {
+            return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE;
+        } else {
+            return isHint ? EntryType.HINT : EntryType.ENTITY;
+        }
+    }
+
+
     private void mergeForceRedactions(Set<ManualForceRedaction> forceRedactions, EntityLog entityLog) {
 
         forceRedactions.forEach(forceRedaction -> {
@@ -312,13 +332,39 @@ public class EntityLogService {
         }
     }
 
-    private boolean isHint(String type, String dossierTemplateId) {
+    private boolean isHint(String type, DossierEntity dossier, FileModel fileStatus) {
 
-        String typeId = toTypeId(type, dossierTemplateId);
+        String typeId = toTypeId(type, dossier.getDossierTemplateId());
         TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId);
+
         if (typeEntity == null) {
-            throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId);
+            var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst();
+            if (optionalType.isPresent()) {
+                typeEntity = optionalType.get();
+            } else {
+                throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId);
+            }
         }
+
+        if (Objects.equals(type, "CBI_address"))  {
+
+            String vertebrateStudyId = "";
+            var fileAttributesMap = fileStatus.getFileAttributes();
+            var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId());
+            var fileAttributeConfigs = dossierTemplate.getFileAttributeConfigs();
+            var vertebrateStudy = fileAttributeConfigs.stream().filter(fileAttributeConfigEntity -> fileAttributeConfigEntity.getLabel().equals("Vertebrate Study")).findFirst();
+
+            if (vertebrateStudy.isPresent()) {
+                vertebrateStudyId = vertebrateStudy.get().getId();
+            }
+
+            if (!Objects.equals(vertebrateStudyId, "")
+                    && fileAttributesMap.containsKey(vertebrateStudyId)
+                    && (fileAttributesMap.get(vertebrateStudyId) == null || !fileAttributesMap.get(vertebrateStudyId).equals("Yes"))) {
+                return true;
+            }
+        }
+
         return typeEntity.isHint();
     }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogTest.java

@@ -18,6 +18,7 @@ import java.util.UUID;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.testcontainers.shaded.com.google.common.collect.Lists;
@@ -32,6 +33,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -73,12 +75,15 @@ public class EntityLogTest {
     @MockBean
     private DictionaryPersistenceService dictionaryPersistenceService;
 
+    @MockBean
+    private DossierTemplatePersistenceService dossierTemplatePersistenceService;
+
     private EntityLogService entityLogService;
 
     @BeforeEach
     public void setUp() {
 
-        entityLogService = new EntityLogService(fileManagementStorageService, fileStatusService, manualRedactionProviderService, unprocessedManualEntityClient, dossierService, commentService, dictionaryPersistenceService);
+        entityLogService = new EntityLogService(fileManagementStorageService, fileStatusService, manualRedactionProviderService, unprocessedManualEntityClient, dossierService, commentService, dictionaryPersistenceService, dossierTemplatePersistenceService);
     }
 
 

persistence-service-v1/pom.xml

@@ -31,10 +31,10 @@
     </modules>
 
     <properties>
-        <redaction-service.version>4.165.0</redaction-service.version>
+        <redaction-service.version>4.177.0</redaction-service.version>
         <search-service.version>2.71.0</search-service.version>
         <pdftron-redaction-service.version>4.38.0</pdftron-redaction-service.version>
-        <redaction-report-service.version>4.33.0</redaction-report-service.version>
+        <redaction-report-service.version>4.36.0</redaction-report-service.version>
         <ocr-service.version>3.10.0</ocr-service.version>
         <storage.commons.version>2.45.0</storage.commons.version>
     </properties>