Pull request #223: Master

Merge in RED/persistence-service from feature/RED-3350 to master

* commit 'ffd4da35adfdb791dbfa34772a2c5a77f820a225':
  RED-3350 Communication with entity recognition via queues

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -1,5 +1,17 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
 
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.transaction.Transactional;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@@ -10,17 +22,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import lombok.RequiredArgsConstructor;
-import org.springframework.stereotype.Service;
 
-import javax.transaction.Transactional;
+import lombok.RequiredArgsConstructor;
-import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 @Service
 @RequiredArgsConstructor
@@ -54,13 +57,12 @@ public class FileStatusPersistenceService {
     public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
                                        long legalBasisVersion, long duration, long dossierDictionaryVersion,
                                        int analysisVersion, int analysisNumber) {
+
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED,
+        fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
-                dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion,
+                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
-                analysisVersion, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
     }
 
 
@@ -75,14 +77,21 @@ public class FileStatusPersistenceService {
     }
 
 
+    private boolean isFileDeleted(String fileId) {
+
+        return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
+    }
+
+
     @Transactional
     public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) {
 
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
+        fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
-                approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null, approval);
+                .truncatedTo(ChronoUnit.MILLIS), approval ? OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS) : null, approval);
 
     }
 
@@ -95,26 +104,15 @@ public class FileStatusPersistenceService {
         }
         if (processingStatus == ProcessingStatus.PROCESSED) {
             // In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
-            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
+            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
-                    OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         } else {
-            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
+                    .truncatedTo(ChronoUnit.MILLIS));
         }
     }
 
 
-    @Transactional
-    public void setUpdateStatusIndexingSuccessful(String fileId) {
-
-        if (isFileDeleted(fileId)) {
-            return;
-        }
-        fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED,
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-    }
-
-
     @Transactional
     public void updateLastOCRTime(String fileId, OffsetDateTime time) {
 
@@ -146,14 +144,13 @@ public class FileStatusPersistenceService {
 
 
     @Transactional
-    public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
+    public void setUpdateStatusIndexingSuccessful(String fileId) {
-                                                              boolean hasSuggestions) {
 
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId,
+        fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
+                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
 
 
@@ -178,8 +175,28 @@ public class FileStatusPersistenceService {
     }
 
 
+    @Transactional
+    public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
+                                                              boolean hasSuggestions) {
+
+        if (isFileDeleted(fileId)) {
+            return;
+        }
+        fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
+    }
+
+
+    public FileEntity getStatus(String fileId) {
+
+        return fileRepository.findById(fileId).orElseThrow(() -> new NotFoundException("Unknown file=" + fileId));
+
+    }
+
+
     @Transactional
     public void setExcludedPages(String fileId, Set<Integer> excludedPages) {
+
         fileRepository.findById(fileId).ifPresentOrElse((file) -> {
             if (file.isSoftOrHardDeleted()) {
                 return;
@@ -195,18 +212,6 @@ public class FileStatusPersistenceService {
     }
 
 
-    public FileEntity getStatus(String fileId) {
-
-        return fileRepository.findById(fileId).orElseThrow(() -> new NotFoundException("Unknown file=" + fileId));
-
-    }
-
-
-    public List<FileEntity> getActiveFiles(String dossierId) {
-        return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList());
-    }
-
-
     public List<FileEntity> getStatusesForDossier(String dossierId) {
 
         return fileRepository.findByDossierId(dossierId);
@@ -219,14 +224,12 @@ public class FileStatusPersistenceService {
     }
 
 
-    @Transactional
+    public List<FileEntity> getActiveFiles(String dossierId) {
-    public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
 
-        int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED,
+        return fileRepository.findByDossierId(dossierId)
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
+                .stream()
-        if (countUpdate == 0) {
+                .filter(f -> !f.isSoftOrHardDeleted())
-            throw new NotFoundException("Unknown file=" + fileId);
+                .collect(Collectors.toList());
-        }
     }
 
 
@@ -243,6 +246,17 @@ public class FileStatusPersistenceService {
     }
 
 
+    @Transactional
+    public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
+
+        int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
+        if (countUpdate == 0) {
+            throw new NotFoundException("Unknown file=" + fileId);
+        }
+    }
+
+
     @Transactional
     public void undelete(String fileId) {
 
@@ -253,8 +267,8 @@ public class FileStatusPersistenceService {
         }, () -> {
             throw new NotFoundException("Unknown file=" + fileId);
         });
-        fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED,
+        fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
+                .truncatedTo(ChronoUnit.MILLIS), null);
     }
 
 
@@ -264,8 +278,8 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover,
+        int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now()
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                .truncatedTo(ChronoUnit.MILLIS));
         if (updateCount == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -274,21 +288,26 @@ public class FileStatusPersistenceService {
 
     @Transactional
     public void toggleExclusion(String fileId, boolean excluded) {
+
         if (isFileDeleted(fileId)) {
             return;
         }
-        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
     }
 
+
     @Transactional
     public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) {
+
         if (isFileDeleted(fileId)) {
             return;
         }
-        int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -302,22 +321,14 @@ public class FileStatusPersistenceService {
             file.setExcludedPages(new HashSet<>());
         });
 
-        int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS,
+        int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now()
-                WorkflowStatus.NEW,
+                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
     }
 
 
-    @Transactional
-    public void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime changeDate) {
-        fileRepository.setLastAnnotationModificationDateForFile(fileId, changeDate);
-    }
-
-
     private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file,
                                                             Map<String, String> fileAttributesMap) {
 
@@ -337,15 +348,23 @@ public class FileStatusPersistenceService {
         }).collect(Collectors.toList());
     }
 
-    private boolean isFileDeleted(String fileId) {
+
-        return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
+    @Transactional
+    public void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime changeDate) {
+
+        fileRepository.setLastAnnotationModificationDateForFile(fileId, changeDate);
     }
 
+
     public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
+
         return fileRepository.existsByDossierIdAndLastUpdatedIsAfter(dossierId, since);
     }
 
+
     public int countSoftDeletedFiles(String dossierId) {
+
         return fileRepository.countSoftDeletedFiles(dossierId);
     }
+
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java

@@ -1,11 +1,12 @@
 package com.iqser.red.service.peristence.v1.server.configuration;
 
-import lombok.RequiredArgsConstructor;
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import lombok.RequiredArgsConstructor;
+
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
@@ -35,6 +36,37 @@ public class MessagingConfiguration {
     public static final String IMAGE_SERVICE_RESPONSE_QUEUE = "image_response_queue";
     public static final String IMAGE_SERVICE_DLQ = "image_dead_letter_queue";
 
+    public static final String NER_SERVICE_QUEUE = "entity_request_queue";
+    public static final String NER_SERVICE_RESPONSE_QUEUE = "entity_response_queue";
+    public static final String NER_SERVICE_DLQ = "entity_dead_letter_queue";
+
+
+    @Bean
+    public Queue nerRequestQueue() {
+
+        return QueueBuilder.durable(NER_SERVICE_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
+                .build();
+    }
+
+
+    @Bean
+    public Queue nerResponseQueue() {
+
+        return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
+                .build();
+    }
+
+
+    @Bean
+    public Queue nerResponseDLQ() {
+
+        return QueueBuilder.durable(NER_SERVICE_DLQ).build();
+    }
+
 
     @Bean
     public Queue imageRequestQueue() {
@@ -55,6 +87,7 @@ public class MessagingConfiguration {
                 .build();
     }
 
+
     @Bean
     public Queue imageResponseDLQ() {
 
@@ -62,7 +95,6 @@ public class MessagingConfiguration {
     }
 
 
-
     @Bean
     public Queue redactionQueue() {
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java

@@ -1,13 +1,17 @@
 package com.iqser.red.service.peristence.v1.server.controller;
 
-import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService;
-import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import lombok.RequiredArgsConstructor;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService;
+import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
 @RestController
 @RequiredArgsConstructor
 public class FileStatusProcessingUpdateController implements FileStatusProcessingUpdateResource {

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/NerServiceRequest.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.peristence.v1.server.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class NerServiceRequest {
+
+    private String dossierId;
+    private String fileId;
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java

@@ -1,5 +1,12 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@@ -8,15 +15,10 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
-import java.io.IOException;
-import java.io.InputStream;
 
 @Slf4j
 @Service
@@ -49,6 +51,7 @@ public class FileManagementStorageService {
 
 
     public void storeObject(String storageId, InputStream data) {
+
         storageService.storeObject(storageId, data);
     }
 
@@ -81,11 +84,25 @@ public class FileManagementStorageService {
         }
     }
 
-    public boolean imageInfoExists(String dossierId, String fileId){
+
+    public boolean imageInfoExists(String dossierId, String fileId) {
+
         return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
     }
 
 
+    public boolean textExists(String dossierId, String fileId) {
+
+        return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+    }
+
+
+    public boolean nerEntitiesExists(String dossierId, String fileId) {
+
+        return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
+    }
+
+
     public void deleteObject(String dossierId, String fileId, FileType fileType) {
 
         storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType));

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
-import com.iqser.red.service.search.v1.model.IndexMessageType;
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.search.v1.model.IndexMessageType;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -21,6 +23,8 @@ public class FileStatusProcessingUpdateService {
     private final DossierPersistenceService dossierPersistenceService;
     private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
     private final ManualRedactionService manualRedactionService;
+    private final FileManagementServiceSettings settings;
+    private final FileManagementStorageService fileManagementStorageService;
 
 
     public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
@@ -29,16 +33,25 @@ public class FileStatusProcessingUpdateService {
 
         switch (analyzeResult.getMessageType()) {
 
+            case STRUCTURE_ANALYSE:
+                if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
+                    fileStatusService.addToNerQueue(dossierId, fileId);
+                } else {
+                    analyzeResult.setMessageType(MessageType.ANALYSE);
+                    analysisSuccessful(dossierId, fileId, analyzeResult);
+                }
+                break;
+
             case SURROUNDING_TEXT:
                 fileStatusService.setStatusProcessed(analyzeResult.getFileId());
                 manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
                 break;
+
+            case ANALYSE:
             case REANALYSE:
-            case FULL_ANALYSE:
             default:
                 retryTemplate.execute(retryContext -> {
-                    log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
+                    log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
-                            .getRetryCount());
                     fileStatusService.setStatusSuccessful(fileId, analyzeResult);
                     return null;
                 });
@@ -57,8 +70,7 @@ public class FileStatusProcessingUpdateService {
     public void ocrSuccessful(String dossierId, String fileId) {
 
         retryTemplate.execute(retryContext -> {
-            log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
+            log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
-                    .getRetryCount());
 
             fileStatusService.updateLastOCRTime(fileId);
             fileStatusService.setStatusFullReprocess(dossierId, fileId, 2);
@@ -103,8 +115,7 @@ public class FileStatusProcessingUpdateService {
     public void indexingSuccessful(String dossierId, String fileId) {
 
         retryTemplate.execute(retryContext -> {
-            log.info("Indexing Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
+            log.info("Indexing Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
-                    .getRetryCount());
             fileStatusService.setStatusIndexingSuccessful(fileId);
             return null;
         });

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -1,36 +1,47 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage;
 import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.peristence.v1.server.controller.RulesController;
+import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
 import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.transaction.Transactional;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
 
 @Slf4j
 @Service
@@ -98,11 +109,30 @@ public class FileStatusService {
 
     public void setStatusSuccessful(String fileId, AnalyzeResult analyzeResult) {
 
-        fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(),
+        fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), analyzeResult.getAnalysisNumber());
-                analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(),
+    }
-                analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(),
+
-                analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(),
+
-                analyzeResult.getAnalysisNumber());
+    @Transactional
+    public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse,
+                                   boolean triggeredManually) {
+
+        log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
+
+        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
+
+        if (fileStatus.isExcluded()) {
+            log.debug("File {} is excluded", fileStatus.getId());
+            return;
+        }
+
+        if (fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually) {
+            log.debug("File {} is excluded from automatic analysis", fileStatus.getId());
+            return;
+        }
+
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
+        addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
     }
 
 
@@ -118,63 +148,6 @@ public class FileStatusService {
     }
 
 
-    @Transactional
-    public void createStatus(String dossierId, String fileId, String uploader, String filename) {
-
-        fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
-        addToAnalysisQueue(dossierId, fileId, 1, Set.of());
-    }
-
-
-    @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, int priority) {
-
-        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
-    }
-
-    @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) {
-
-        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
-    }
-
-
-    @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
-
-        log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
-
-        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
-
-        if (fileStatus.isExcluded()) {
-            log.debug("File {} is excluded", fileStatus.getId());
-            return;
-        }
-
-        if(fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually){
-            log.debug("File {} is excluded from automatic analysis", fileStatus.getId());
-            return;
-        }
-
-        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
-        addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
-    }
-
-
-    public void setStatusOcrProcessing(String dossierId, String fileId) {
-
-        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
-
-        if (fileStatus.isExcluded()) {
-            log.debug("File {} is excluded", fileId);
-            return;
-        }
-
-        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
-        addToOcrQueue(dossierId, fileId, 2);
-    }
-
-
     @Transactional
     public void setStatusFullReprocess(String dossierId, String fileId, int priority) {
 
@@ -186,10 +159,119 @@ public class FileStatusService {
         }
 
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
+
+        log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
+
         addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
     }
 
 
+    @Transactional
+    @SuppressWarnings("PMD")
+    protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
+
+        var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
+        var fileStatus = fileStatusPersistenceService.getStatus(fileId);
+
+        if (fileStatus.isExcluded()) {
+            log.debug("File {} is excluded", fileStatus.getId());
+            return;
+        }
+
+        boolean reanalyse = isReanalyse(dossier, fileStatus);
+
+        if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
+            log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
+            addToImageQueue(dossierId, fileId);
+            return;
+        }
+
+        MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
+
+        if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
+            log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
+            addToNerQueue(dossierId, fileId);
+            return;
+        }
+
+        var analyseRequest = AnalyzeRequest.builder()
+                .messageType(messageType)
+                .dossierId(dossierId)
+                .analysisNumber(fileStatus.getNumberOfAnalyses() + 1)
+                .sectionsToReanalyse(sectionsToReanalyse)
+                .fileId(fileId)
+                .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
+                .dossierTemplateId(dossier.getDossierTemplateId())
+                .lastProcessed(fileStatus.getLastProcessed())
+                .fileAttributes(convert(fileStatus.getFileAttributes()))
+                .excludedPages(fileStatus.getExcludedPages())
+                .build();
+
+        log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
+
+        setStatusProcessing(fileId);
+
+        try {
+            rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
+                message.getMessageProperties().setPriority(priority);
+                return message;
+            });
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) {
+
+        return !fileStatus.getProcessingStatus()
+                .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
+                .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
+                .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
+                .isAfter(fileStatus.getLastFileAttributeChange()));
+    }
+
+
+    private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
+                                             ProcessingStatus processingStatus) {
+
+        if (!fileManagementStorageService.textExists(dossierId, fileId)) {
+            return MessageType.STRUCTURE_ANALYSE;
+        }
+        if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
+            return MessageType.ANALYSE;
+        }
+        if (reanalyse) {
+            return MessageType.REANALYSE;
+        }
+        return MessageType.ANALYSE;
+    }
+
+
+    @Transactional
+    public void setStatusReprocess(String dossierId, String fileId, int priority) {
+
+        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
+    }
+
+
+    @Transactional
+    public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) {
+
+        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
+    }
+
+
+    @Transactional
+    public void createStatus(String dossierId, String fileId, String uploader, String filename) {
+
+        fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
+        addToAnalysisQueue(dossierId, fileId, 1, Set.of());
+    }
+
+
     public void setStatusIndexingSuccessful(String fileId) {
 
         fileStatusPersistenceService.setUpdateStatusIndexingSuccessful(fileId);
@@ -259,53 +341,23 @@ public class FileStatusService {
     }
 
 
-    @Transactional
+    public void setStatusOcrProcessing(String dossierId, String fileId) {
-    @SuppressWarnings("PMD")
-    protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
 
-        var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
+        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
-        var fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
         if (fileStatus.isExcluded()) {
-            log.debug("File {} is excluded", fileStatus.getId());
+            log.debug("File {} is excluded", fileId);
             return;
         }
 
-        var fileAttributes = fileStatus.getFileAttributes();
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
+        addToOcrQueue(dossierId, fileId, 2);
+    }
 
-        boolean reanalyse = !fileStatus.getProcessingStatus()
-                .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
-                .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
-                .isAfter(fileStatus.getLastFileAttributeChange()));
 
-        if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
+    public void setStatusNerAnalyzing(String fileId) {
-            addToImageQueue(dossierId, fileId);
-            return;
-        }
 
-        var analyseRequest = AnalyzeRequest.builder()
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING);
-                .messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE)
-                .dossierId(dossierId)
-                .analysisNumber(fileStatus.getNumberOfAnalyses()+1)
-                .sectionsToReanalyse(sectionsToReanalyse)
-                .fileId(fileId)
-                .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
-                .dossierTemplateId(dossier.getDossierTemplateId())
-                .lastProcessed(fileStatus.getLastProcessed())
-                .fileAttributes(convert(fileAttributes))
-                .excludedPages(fileStatus.getExcludedPages())
-                .build();
-
-        setStatusProcessing(fileId);
-
-        try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
-                message.getMessageProperties().setPriority(priority);
-                return message;
-            });
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-        }
     }
 
 
@@ -427,6 +479,35 @@ public class FileStatusService {
     }
 
 
+    @Transactional
+    public void setStatusAnalyse(String dossierId, String fileId, int priority) {
+
+        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
+
+        if (fileStatus.isExcluded()) {
+            log.debug("File {} is excluded", fileStatus.getId());
+            return;
+        }
+
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
+        addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
+    }
+
+
+    protected void addToNerQueue(String dossierId, String fileId) {
+
+        setStatusNerAnalyzing(fileId);
+        try {
+            rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(new NerServiceRequest(dossierId, fileId)), message -> {
+                message.getMessageProperties().setPriority(1);
+                return message;
+            });
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
     public int countSoftDeletedFiles(String dossierId) {
 
         return fileStatusPersistenceService.countSoftDeletedFiles(dossierId);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ImageMessageReceiver.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.peristence.v1.server.service;
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.Set;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
@@ -39,7 +38,7 @@ public class ImageMessageReceiver {
         String fileId = (String) imageResponse.get("fileId");
 
         fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, in);
-        fileStatusService.setStatusFullReprocess(dossierId, fileId, 1);
+        fileStatusService.setStatusAnalyse(dossierId, fileId, 1);
     }
 
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/NerMessageReceiver.java

@@ -0,0 +1,55 @@
+package com.iqser.red.service.peristence.v1.server.service;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class NerMessageReceiver {
+
+    private final FileStatusService fileStatusService;
+    private final ObjectMapper objectMapper;
+    private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
+
+
+    @SneakyThrows
+    @RabbitListener(queues = MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE)
+    public void receive(byte[] in) {
+
+        HashMap<String, Object> entityResponse = objectMapper.readValue(in, new TypeReference<>() {
+        });
+
+        String dossierId = (String) entityResponse.get("dossierId");
+        String fileId = (String) entityResponse.get("fileId");
+
+        log.info("Received NER Message from {} {}", dossierId, fileId);
+        fileStatusService.addToAnalysisQueue(dossierId, fileId, 2, null);
+    }
+
+
+    @RabbitListener(queues = MessagingConfiguration.NER_SERVICE_DLQ)
+    public void handleDLQMessage(Message failedMessage) throws IOException {
+
+        HashMap<String, Object> entityResponse = objectMapper.readValue(failedMessage.getBody(), new TypeReference<>() {
+        });
+        String dossierId = (String) entityResponse.get("dossierId");
+        String fileId = (String) entityResponse.get("fileId");
+
+        log.info("Received NER DLQ Message from {} {}", dossierId, fileId);
+        fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId);
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java

@@ -1,9 +1,11 @@
 package com.iqser.red.service.peristence.v1.server.settings;
 
-import com.iqser.red.service.persistence.management.v1.processor.settings.FileUploadSettings;
-import lombok.Data;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileUploadSettings;
+
+import lombok.Data;
+
 @Data
 @ConfigurationProperties("persistence-service")
 public class FileManagementServiceSettings {
@@ -22,4 +24,5 @@ public class FileManagementServiceSettings {
     private boolean migrateOnly;
 
     private boolean imageServiceEnabled = true;
+    private boolean nerServiceEnabled = true;
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java

@@ -1,5 +1,10 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
 import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
@@ -11,10 +16,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import static org.assertj.core.api.Assertions.assertThat;
 
 public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
@@ -49,7 +50,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
                 .manualRedactions(new ManualRedactions())
-                .messageType(MessageType.FULL_ANALYSE)
+                .messageType(MessageType.ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())
                 .dossierId(dossier.getId())
@@ -99,7 +100,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
                 .manualRedactions(new ManualRedactions())
-                .messageType(MessageType.FULL_ANALYSE)
+                .messageType(MessageType.ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())
                 .dossierId(dossier.getId())
@@ -141,7 +142,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
                 .manualRedactions(new ManualRedactions())
-                .messageType(MessageType.FULL_ANALYSE)
+                .messageType(MessageType.ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())
                 .dossierId(dossier.getId())

persistence-service-v1/pom.xml

@@ -25,7 +25,7 @@
     </modules>
 
     <properties>
-        <redaction-service.version>3.72.0</redaction-service.version>
+        <redaction-service.version>3.74.0</redaction-service.version>
         <search-service.version>2.18.0</search-service.version>
         <pdftron-redaction-service.version>3.31.0</pdftron-redaction-service.version>
         <redaction-report-service.version>3.19.0</redaction-report-service.version>