Merge branch 'RED-10127' into 'master'

RED-10127: enable ocr rerun via support controller

Closes RED-10127

See merge request redactmanager/persistence-service!787

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java

@@ -272,7 +272,7 @@ public class FileManagementController implements FileManagementResource {
 //                if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
 //                    reanalysisService.ocrFile(dossierId, fileId, true);
 //                } else {
-                    fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+                    fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
 //                }
             }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -60,18 +60,20 @@ public class SupportController implements SupportResource {
 
 
     @Override
-    public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
+    public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                               @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
 
-        reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis);
+        reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis, runOcr);
     }
 
 
     @Override
     public void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
                                                   @RequestBody List<String> fileIds,
-                                                  @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
+                                                  @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                                  @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
 
-        reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis);
+        reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis, runOcr);
 
     }
 
@@ -156,6 +158,7 @@ public class SupportController implements SupportResource {
         return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
+
     @Override
     @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
     public ImportResponse importDataset(MultipartFile file) {

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -51,6 +51,7 @@ public interface SupportResource {
     String FALSE = "false";
 
     String FULL_REANALYSIS_PARAM = "fullReanalysis";
+    String RUN_OCR_PARAM = "runOcr";
     String EXPORT = "/export";
     String IMPORT = "/import";
 
@@ -80,7 +81,8 @@ public interface SupportResource {
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH)
     @Operation(summary = "Reanalyze all files in error state.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
+    void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                        @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
 
 
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH)
@@ -88,7 +90,8 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
                                            @RequestBody List<String> fileIds,
-                                           @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
+                                           @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                           @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -146,9 +149,7 @@ public interface SupportResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
-    @PostMapping(value = DATASET_EXCHANGE
+    @PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-                         + EXPORT
-                         + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
             ## Export Preview Files Endpoint
             
@@ -166,6 +167,7 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java

@@ -37,7 +37,7 @@ public class AdminInterfaceController {
 
         fileManagementStorageService.deleteAllObjects(dossierId, fileId);
 
-        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
 
     }
 
@@ -136,7 +136,7 @@ public class AdminInterfaceController {
                         fileManagementStorageService.deleteEntityLog(dossierId, fileId);
                         fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
 
-                        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+                        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
                     }
 
                 }

persistence-service-v1/persistence-service-processor-v1/build.gradle.kts

@@ -38,10 +38,7 @@ dependencies {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
-    api("com.knecon.fforesight.service:ocr-service-api:4.24.0") {
+    api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
-    }
     implementation("com.knecon.fforesight:llm-service-api:1.17.0")
     api("com.knecon.fforesight:jobs-commons:0.10.0")
     api("com.iqser.red.commons:storage-commons:2.50.0")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java

@@ -78,8 +78,8 @@ public class DatasetExchangeService {
         var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
         String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
         String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
-        List<FileEntity> files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId);
+        List<FileEntity> files = fileStatusPersistenceService.getActiveFilesForDossierTemplate(dossierTemplateId);
-        List<DossierEntity> dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
+        List<DossierEntity> dossiers = dossierPersistenceService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId);
         if (dossiers.isEmpty()) {
             throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java

@@ -58,7 +58,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration  {
                 files.forEach(file -> {
                     if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
                         log.info("Set full reanalyse for file {}", file.getId());
-                        fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true);
+                        fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true, false);
                         log.info("Finished migration of file {}", file.getId());
                     }
                 });

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 import org.apache.commons.lang3.StringUtils;
 
 import org.springframework.retry.support.RetryTemplate;
-import org.springframework.scheduling.annotation.Async;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@@ -160,7 +159,7 @@ public class FileStatusProcessingUpdateService {
         retryTemplate.execute(retryContext -> {
             log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
 
-            fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true);
+            fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true, false);
 
             return null;
         });

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -5,13 +5,14 @@ import static com.iqser.red.service.persistence.management.v1.processor.configur
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
+
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Value;
@@ -76,6 +77,7 @@ import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverte
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.knecon.fforesight.llm.service.LlmNerMessage;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
 import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
@@ -144,6 +146,7 @@ public class FileStatusService {
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
     }
 
+
     public List<DossierStatsFileProjection> getDossierStatsFiles(String dossierId) {
 
         return fileStatusPersistenceService.getFilesForDossierStats(dossierId);
@@ -718,13 +721,26 @@ public class FileStatusService {
     public void addToOcrQueue(String dossierId, String fileId, int priority) {
 
         var removeWatermark = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)).isRemoveWatermark();
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE,
+        Set<AzureOcrFeature> features = new HashSet<>();
-                                      TenantContext.getTenantId(),
+        if (removeWatermark) {
-                                      new DocumentRequest(dossierId, fileId, removeWatermark),
+            features.add(AzureOcrFeature.REMOVE_WATERMARKS);
-                                      message -> {
+        }
-                                          message.getMessageProperties().setPriority(priority);
+        if (applicationType.equals("DocuMine")) {
-                                          return message;
+            features.add(AzureOcrFeature.ROTATION_CORRECTION);
-                                      });
+            features.add(AzureOcrFeature.FONT_STYLE_DETECTION);
+        }
+
+        var request = DocumentRequest.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) // needed for legacy OCR-services
+                .features(features)
+                .build();
+
+        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE, TenantContext.getTenantId(), request, message -> {
+            message.getMessageProperties().setPriority(priority);
+            return message;
+        });
     }
 
 
@@ -893,13 +909,13 @@ public class FileStatusService {
         websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
 
         if (!excluded) {
-            setStatusFullReprocess(dossierId, fileId, false, true);
+            setStatusFullReprocess(dossierId, fileId, false, true, false);
         }
     }
 
 
     @Transactional
-    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
+    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr) {
 
         FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
@@ -908,11 +924,17 @@ public class FileStatusService {
             return;
         }
 
-        if (requiresStructureAnalysis) {
+        if (requiresStructureAnalysis || runOcr) {
             log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
             fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
         }
 
+        if (runOcr) {
+            fileStatusPersistenceService.resetOcrStartAndEndDate(fileId);
+            setStatusOcrQueued(dossierId, fileId);
+            return;
+        }
+
         addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT);
     }
 
@@ -925,7 +947,7 @@ public class FileStatusService {
         websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
 
         if (!excludedFromAutomaticAnalysis) {
-            setStatusFullReprocess(dossierId, fileId, false, true);
+            setStatusFullReprocess(dossierId, fileId, false, true, false);
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java

@@ -48,10 +48,10 @@ public class ReanalysisService {
     private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
 
 
-    public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis) {
+    public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis, boolean runOcr) {
 
         var errorFiles = fileStatusService.getAllRelevantErrorFiles();
-        analyseFiles(repeatStructureAnalysis, errorFiles);
+        analyseFiles(repeatStructureAnalysis, errorFiles, runOcr);
     }
 
 
@@ -88,7 +88,7 @@ public class ReanalysisService {
         if (force) {
             filesToReanalyse.forEach(file -> {
                 if (file.isFullAnalysisRequired()) {
-                    fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false);
+                    fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false);
                 } else {
                     fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true);
                 }
@@ -98,7 +98,7 @@ public class ReanalysisService {
                     .filter(FileModel::isReanalysisRequired)
                     .forEach(file -> {
                         if (file.isFullAnalysisRequired()) {
-                            fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false);
+                            fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false);
                         } else {
                             fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true);
                         }
@@ -107,9 +107,9 @@ public class ReanalysisService {
     }
 
 
-    private void analyseFiles(boolean repeatStructureAnalysis, List<FileModel> filesToReanalyse) {
+    private void analyseFiles(boolean repeatStructureAnalysis, List<FileModel> filesToReanalyse, boolean runOcr) {
 
-        filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis));
+        filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis, runOcr));
     }
 
 
@@ -120,9 +120,9 @@ public class ReanalysisService {
     }
 
 
-    public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set<String> fileIds, boolean repeatStructureAnalysis) {
+    public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set<String> fileIds, boolean repeatStructureAnalysis, boolean runOcr) {
 
-        analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds));
+        analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds), runOcr);
     }
 
 
@@ -231,7 +231,7 @@ public class ReanalysisService {
         } catch (Exception e) {
             throw new BadRequestException("File Not Valid PDF or number of pages do not match");
         }
-        fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false);
+        fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false, false);
     }
 
 
@@ -245,7 +245,7 @@ public class ReanalysisService {
                                                      deleteImportedRedactionsRequest.getFileId(),
                                                      FileType.IMPORTED_REDACTIONS,
                                                      importedRedactions);
-        fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false);
+        fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false, false);
     }
 
 
@@ -263,7 +263,7 @@ public class ReanalysisService {
         fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
 
         if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
-            fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false);
+            fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false, false);
         }
     }
 
@@ -281,7 +281,7 @@ public class ReanalysisService {
 
         List<FileModel> rejectedFiles = filterInvalidFiles(files);
 
-        files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis()));
+        files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis(), reanalysisSettings.runOcr()));
 
         return rejectedFiles;
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java

@@ -125,7 +125,7 @@ public class AutomaticAnalysisJob implements Job {
 
                     if (file.isFullAnalysisRequired()) {
                         log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getId());
-                        fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false);
+                        fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false, false);
                     } else if (file.isReanalysisRequired()) {
                         log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getId());
                         fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -194,6 +194,11 @@ public class DossierPersistenceService {
         return dossierRepository.findByDossierTemplateId(dossierTemplateId);
     }
 
+    public List<DossierEntity> findAllActiveDossiersForDossierTemplateId(String dossierTemplateId) {
+
+        return dossierRepository.findActiveByDossierTemplateId(dossierTemplateId);
+    }
+
 
     public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -414,6 +414,16 @@ public class FileStatusPersistenceService {
                 .toList();
     }
 
+    public List<FileEntity> getActiveFilesForDossierTemplate(String dossierTemplateId) {
+
+        return dossierService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId)
+                .stream()
+                .map(dossier -> fileRepository.findByDossierId(dossier.getId()))
+                .flatMap(List::stream)
+                .filter(f -> !f.isSoftOrHardDeleted())
+                .toList();
+    }
+
 
     public List<FileEntity> getStatusesAddedBefore(OffsetDateTime end) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java

@@ -115,4 +115,7 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
     Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
 
+    @Query("select d from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null")
+    List<DossierEntity> findActiveByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
+
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java

@@ -65,35 +65,35 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
         var dossier = dossierTesterAndProvider.provideTestDossier();
         var file = fileTesterAndProvider.testAndProvideFile(dossier);
 
-        var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
+        var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
         assertThat(e.status()).isEqualTo(409);
 
         setProcessingStatusToError(dossier, file);
 
-        supportClient.reanalyzeAllRelevantErrorFiles(true);
+        supportClient.reanalyzeAllRelevantErrorFiles(true, false);
         var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
         setProcessingStatusToError(dossier, file);
 
-        supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true);
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true, false);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
         setProcessingStatusToError(dossier, file);
 
-        supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true);
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
         setProcessingStatusToError(dossier, file);
 
         setFileToStatusDeleted(file);
-        e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
+        e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
         assertThat(e.status()).isEqualTo(400);
 
         setFileToStatusHardDeleted(file);
-        e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
+        e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false));
         assertThat(e.status()).isEqualTo(400);
 
     }
@@ -177,7 +177,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
 
         dossierClient.deleteDossier(dossier1.getId());
 
-        supportClient.reanalyzeAllRelevantErrorFiles(false);
+        supportClient.reanalyzeAllRelevantErrorFiles(false, false);
         FeignException feignException = assertThrows(FeignException.class, () -> fileClient.getFileStatus(dossier1.getId(), file1.getId()));
         assertEquals(feignException.status(), 404);
         FileStatus fileStatus = FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(file1.getId()));
@@ -187,11 +187,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
 
         setProcessingStatusToError(dossier2, file2);
 
-        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false);
         loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
         assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
-        feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
+        feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false));
         assertEquals(feignException.status(), 404);
 
         setProcessingStatusToError(dossier2, file2);
@@ -200,6 +200,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
                                                                                      new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
                                                                                                             Collections.emptySet(),
                                                                                                             true,
+                                                                                                            false,
                                                                                                             new FileStatusFilter(null, null, true, true)));
         loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
         assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
@@ -224,7 +225,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
 
         dossierClient.archiveDossiers(Collections.singleton(dossier1.getId()));
 
-        supportClient.reanalyzeAllRelevantErrorFiles(false);
+        supportClient.reanalyzeAllRelevantErrorFiles(false, false);
         FileStatus fileStatus = fileClient.getFileStatus(dossier1.getId(), file1.getId());
         assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
         var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
@@ -232,11 +233,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
 
         setProcessingStatusToError(dossier2, file2);
 
-        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false);
         loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
         assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
-        FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
+        FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false));
         assertEquals(feignException.status(), 404);
 
         setProcessingStatusToError(dossier2, file2);
@@ -245,6 +246,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
                                                                                      new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
                                                                                                             Collections.emptySet(),
                                                                                                             true,
+                                                                                                            false,
                                                                                                             null));
         loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
         assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java

@@ -9,6 +9,7 @@ public record ReanalysisSettings(
         @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set<String> dossierIds,
         @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set<String> fileIds,
         @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis,
+        @Schema(description = "If set to true, ocr will be repeated and therefore also layout parsing and named entity recognition.", defaultValue = "false") boolean runOcr,
         @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter
 ) {