Merge branch 'RED-9205-fix' into 'release/2.262.x'

RED-9205 - Fix download job failing

See merge request redactmanager/persistence-service!499

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/ReanalyzeNonApprovedFilesService.java

@@ -0,0 +1,55 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
+import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ReanalyzeNonApprovedFilesService {
+
+    private final DossierRepository dossierRepository;
+    private final FileRepository fileRepository;
+    private final ReanalysisService reanalysisService;
+
+    private final LayoutParsingRequestFactory layoutParsingRequestFactory;
+    private final RabbitTemplate rabbitTemplate;
+
+
+    public void reanalyzeNonApprovedFiles() {
+
+        List<DossierEntity> dossierEntities = dossierRepository.findAll();
+        dossierEntities.stream()
+                .filter(dossier -> !dossier.isDeleted())
+                .forEach(dossierEntity -> {
+                    Set<String> unapprovedFilesByDossierIds = fileRepository.findByDossierIdAndWorkflowStatusNotApproved(dossierEntity.getId())
+                            .stream()
+                            .filter(fileEntity -> !fileEntity.isSoftOrHardDeleted())
+                            .map(FileEntity::getId)
+                            .collect(Collectors.toSet());
+
+                    unapprovedFilesByDossierIds.forEach(id -> {
+                        LayoutParsingRequest layoutParsingRequest = layoutParsingRequestFactory.build(dossierEntity.getId(), id, false);
+                        rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
+                    });
+
+                    reanalysisService.reanalyzeFiles(dossierEntity.getId(), unapprovedFilesByDossierIds, true);
+                });
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReanalyzeNonApprovedFilesMigration18.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.migration.ReanalyzeNonApprovedFilesService;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class ReanalyzeNonApprovedFilesMigration18 extends Migration {
+
+    @Autowired
+    private ReanalyzeNonApprovedFilesService reanalyzeNonApprovedFilesService;
+
+    private static final String NAME = "Reanalyze all non-approved files";
+    private static final long VERSION = 18;
+
+
+    public ReanalyzeNonApprovedFilesMigration18() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Reanalyzing all non-approved files.");
+        reanalyzeNonApprovedFilesService.reanalyzeNonApprovedFiles();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java

@@ -84,7 +84,7 @@ public class ReanalysisRequiredStatusService {
         }
 
         if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) {
-            return new AnalysisRequiredResult(false, true);
+            return new AnalysisRequiredResult(false, false);
         }
 
         if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.PRE_PROCESSED.equals(fileStatus.getProcessingStatus()) || ignoreProcessingStates) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java

@@ -37,6 +37,7 @@ import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -46,21 +47,24 @@ import java.util.stream.Collectors;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class DownloadPreparationService {
 
-    DownloadStatusPersistenceService downloadStatusPersistenceService;
+    private final DownloadStatusPersistenceService downloadStatusPersistenceService;
-    FileManagementStorageService fileManagementStorageService;
+    private final FileManagementStorageService fileManagementStorageService;
-    ReportTemplatePersistenceService reportTemplatePersistenceService;
+    private final ReportTemplatePersistenceService reportTemplatePersistenceService;
-    NotificationPersistenceService notificationPersistenceService;
+    private final NotificationPersistenceService notificationPersistenceService;
-    RabbitTemplate rabbitTemplate;
+    private final RabbitTemplate rabbitTemplate;
-    ObjectMapper objectMapper;
+    private final ObjectMapper objectMapper;
-    DownloadReportCleanupService downloadReportCleanupService;
+    private final DownloadReportCleanupService downloadReportCleanupService;
-    ColorsService colorsService;
+    private final ColorsService colorsService;
-    FileManagementServiceSettings settings;
+    private final FileManagementServiceSettings settings;
-    DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
-    DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
+    private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
 
+    @Value("${storage.backend}")
+    private String storageBackend;
+
+    private static final String REPORT_INFO = "/REPORT_INFO.json";
 
     @Transactional
     public void createDownload(ReportResultMessage reportResultMessage) {
@@ -321,13 +325,31 @@ public class DownloadPreparationService {
     @SneakyThrows
     private List<StoredFileInformation> getStoredFileInformation(String downloadId) {
 
-        var storageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
+        String storageId;
+        if (storageBackend.equals("s3")) {
+            storageId = generateReportJsonStorageIdForS3(downloadId);
+        } else {
+            storageId = generateReportJsonStorageIdForAzure(downloadId);
+        }
 
         return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
         });
     }
 
 
+
+    private String generateReportJsonStorageIdForS3(String storageId) {
+
+        return storageId.substring(0, storageId.length() - 3) + REPORT_INFO;
+    }
+
+
+    private String generateReportJsonStorageIdForAzure(String storageId) {
+
+        return storageId.substring(0, storageId.length() - 4) + REPORT_INFO;
+    }
+
+
     private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) {
 
         if (fileStatus != null) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java

@@ -41,7 +41,7 @@ public class DownloadReadyJob implements Job {
 
                 int numberOfFiles = download.getFiles().size();
                 var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId());
-                if (downloadRedactionFileStatus.size() == numberOfFiles) {
+                if (downloadRedactionFileStatus.size() >= numberOfFiles) {
                     downloadPreparationService.createDownload(downloadRedactionFileStatus, download.getStorageId());
                     downloadPreparationService.clearRedactionStatusEntries(download.getStorageId());
                 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -7,6 +7,7 @@ import java.util.Optional;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
@@ -28,6 +29,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     List<FileEntity> findByAddedBefore(OffsetDateTime end);
 
 
+    @Query("SELECT f FROM FileEntity f WHERE f.dossierId = :dossierId AND f.workflowStatus <> 'APPROVED'")
+    List<FileEntity> findByDossierIdAndWorkflowStatusNotApproved(@Param("dossierId") String dossierId);
+
+
     @Modifying
     @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId")
     void updateFlags(String fileId,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java

@@ -10,7 +10,6 @@ import org.springframework.data.jpa.repository.Query;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 
 public interface RemoveRedactionRepository extends JpaRepository<IdRemovalEntity, AnnotationEntityId>, AnnotationEntityRepository {
@@ -24,13 +23,15 @@ public interface RemoveRedactionRepository extends JpaRepository<IdRemovalEntity
     @Query("update IdRemovalEntity idr set idr.status = :annotationStatus where idr.id = :annotationEntityId")
     void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus);
 
+
     @Query("select idr from IdRemovalEntity idr where idr.id = :annotationEntityId and idr.softDeletedTime is null")
     Optional<IdRemovalEntity> findByIdAndNotSoftDeleted(AnnotationEntityId annotationEntityId);
 
 
-    @Query("select idr from IdRemovalEntity idr where idr.id.fileId = :fileId and (:includeDeletions = true or idr.softDeletedTime is null)")
+    @Query("select idr from IdRemovalEntity idr left join fetch idr.typeIdsOfModifiedDictionaries where idr.id.fileId = :fileId and (:includeDeletions = true or idr.softDeletedTime is null)")
     List<IdRemovalEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
 
+
     @Query("select idr from IdRemovalEntity idr where idr.id.fileId = :fileId and idr.processedDate is null")
     List<IdRemovalEntity> findByFileIdAndUnprocessed(String fileId);
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalyzeNonApprovedFilesTest.java

@@ -0,0 +1,82 @@
+package com.iqser.red.service.peristence.v1.server.integration.tests;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.persistence.management.v1.processor.migration.MigrationController;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
+
+public class ReanalyzeNonApprovedFilesTest extends AbstractPersistenceServerServiceTest {
+
+    @Autowired
+    private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
+
+    @Autowired
+    private FileTesterAndProvider fileTesterAndProvider;
+
+    @Autowired
+    private DossierTesterAndProvider dossierTesterAndProvider;
+
+    @Autowired
+    private ReanalysisService reanalysisService;
+
+    @Autowired
+    private MigrationController migrationController;
+
+    @Autowired
+    private FileStatusManagementService fileStatusManagementService;
+
+
+    @Test
+    public void testReanalyzeNonApprovedFiles() {
+
+        var filename1 = "test-dossier1-approved";
+        var filename2 = "test-dossier1-not-approved";
+        var filename3 = "test-dossier2-approved";
+        var filename4 = "test-dossier2-not-approved";
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier1 = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "dossier1");
+        var dossier2 = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "dossier2");
+
+        var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, filename1);
+        var file2 = fileTesterAndProvider.testAndProvideFile(dossier1, filename2);
+        var file3 = fileTesterAndProvider.testAndProvideFile(dossier2, filename3);
+        var file4 = fileTesterAndProvider.testAndProvideFile(dossier2, filename4);
+
+        fileStatusManagementService.setStatusApproved(dossier1.getId(), file1.getFileId(), "user");
+        fileStatusManagementService.setStatusApproved(dossier2.getId(), file3.getFileId(), "user");
+
+        var fileStatus1 = fileStatusManagementService.getFileStatus(file1.getFileId());
+        var fileStatus2 = fileStatusManagementService.getFileStatus(file2.getFileId());
+        var fileStatus3 = fileStatusManagementService.getFileStatus(file3.getFileId());
+        var fileStatus4 = fileStatusManagementService.getFileStatus(file4.getFileId());
+
+        assertEquals(fileStatus1.getWorkflowStatus(), WorkflowStatus.APPROVED);
+        assertEquals(fileStatus2.getWorkflowStatus(), WorkflowStatus.NEW);
+        assertEquals(fileStatus3.getWorkflowStatus(), WorkflowStatus.APPROVED);
+        assertEquals(fileStatus4.getWorkflowStatus(), WorkflowStatus.NEW);
+
+        migrationController.run("Reanalyze all non-approved files", true);
+
+        fileStatus1 = fileStatusManagementService.getFileStatus(file1.getFileId());
+        fileStatus2 = fileStatusManagementService.getFileStatus(file2.getFileId());
+        fileStatus3 = fileStatusManagementService.getFileStatus(file3.getFileId());
+        fileStatus4 = fileStatusManagementService.getFileStatus(file4.getFileId());
+
+        assertEquals(fileStatus1.getProcessingStatus(), ProcessingStatus.PRE_PROCESSING_QUEUED);
+        assertEquals(fileStatus2.getProcessingStatus(), ProcessingStatus.FULL_PROCESSING);
+        assertEquals(fileStatus3.getProcessingStatus(), ProcessingStatus.PRE_PROCESSING_QUEUED);
+        assertEquals(fileStatus4.getProcessingStatus(), ProcessingStatus.FULL_PROCESSING);
+    }
+
+}