diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java index 2dfc208cf..b1123932e 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java @@ -272,7 +272,7 @@ public class FileManagementController implements FileManagementResource { // if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) { // reanalysisService.ocrFile(dossierId, fileId, true); // } else { - fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false); // } } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index e0b802fce..ba626cdd6 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -60,18 +60,20 @@ public class SupportController implements SupportResource { @Override - public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) { + public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis, + @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) { - reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis); + reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis, runOcr); } @Override public void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody List fileIds, - @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) { + @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis, + @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) { - reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis); + reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis, runOcr); } @@ -156,6 +158,7 @@ public class SupportController implements SupportResource { return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes); } + @Override @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')") public ImportResponse importDataset(MultipartFile file) { diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 5782e030f..899315770 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -51,6 +51,7 @@ public interface SupportResource { String FALSE = "false"; String FULL_REANALYSIS_PARAM = "fullReanalysis"; + String RUN_OCR_PARAM = "runOcr"; String EXPORT = "/export"; String IMPORT = "/import"; @@ -80,7 +81,8 @@ public interface SupportResource { @PostMapping(value = ERROR_REANALYSIS_REST_PATH) @Operation(summary = "Reanalyze all files in error state.", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) - void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis); + void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis, + @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr); @PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH) @@ -88,7 +90,8 @@ public interface SupportResource { @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")}) void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody List fileIds, - @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis); + @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis, + @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr); @ResponseStatus(value = HttpStatus.OK) @@ -146,9 +149,7 @@ public interface SupportResource { @ResponseStatus(value = HttpStatus.OK) @ResponseBody - @PostMapping(value = DATASET_EXCHANGE - + EXPORT - + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) + @PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """ ## Export Preview Files Endpoint @@ -166,6 +167,7 @@ public interface SupportResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file); + @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java index 8e6b10b12..2be0e938e 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java @@ -37,7 +37,7 @@ public class AdminInterfaceController { fileManagementStorageService.deleteAllObjects(dossierId, fileId); - fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false); } @@ -136,7 +136,7 @@ public class AdminInterfaceController { fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId); - fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index ac184a95e..693b8ad65 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -38,10 +38,7 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } - api("com.knecon.fforesight.service:ocr-service-api:4.24.0") { - exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") - exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") - } + api("com.knecon.fforesight:azure-ocr-service-api:0.13.0") implementation("com.knecon.fforesight:llm-service-api:1.17.0") api("com.knecon.fforesight:jobs-commons:0.10.0") api("com.iqser.red.commons:storage-commons:2.50.0") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java index ab765e3c6..23cf9992d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java @@ -78,8 +78,8 @@ public class DatasetExchangeService { var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId); String downloadFilename = template.getName() + "_PREVIEW_FILES.zip"; String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES"); - List files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId); - List dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); + List files = fileStatusPersistenceService.getActiveFilesForDossierTemplate(dossierTemplateId); + List dossiers = dossierPersistenceService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId); if (dossiers.isEmpty()) { throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java index 55df55201..3e7f0db60 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java @@ -58,7 +58,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { log.info("Set full reanalyse for file {}", file.getId()); - fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true); + fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true, false); log.info("Finished migration of file {}", file.getId()); } }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 0b165653e..190b23550 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; -import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -160,7 +159,7 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); - fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true); + fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true, false); return null; }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index bfeab9e79..c924b209b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -5,13 +5,14 @@ import static com.iqser.red.service.persistence.management.v1.processor.configur import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection; + import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Value; @@ -76,6 +77,7 @@ import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverte import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.knecon.fforesight.llm.service.LlmNerMessage; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; +import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature; import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -144,6 +146,7 @@ public class FileStatusService { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); } + public List getDossierStatsFiles(String dossierId) { return fileStatusPersistenceService.getFilesForDossierStats(dossierId); @@ -718,13 +721,26 @@ public class FileStatusService { public void addToOcrQueue(String dossierId, String fileId, int priority) { var removeWatermark = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)).isRemoveWatermark(); - rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE, - TenantContext.getTenantId(), - new DocumentRequest(dossierId, fileId, removeWatermark), - message -> { - message.getMessageProperties().setPriority(priority); - return message; - }); + Set features = new HashSet<>(); + if (removeWatermark) { + features.add(AzureOcrFeature.REMOVE_WATERMARKS); + } + if (applicationType.equals("DocuMine")) { + features.add(AzureOcrFeature.ROTATION_CORRECTION); + features.add(AzureOcrFeature.FONT_STYLE_DETECTION); + } + + var request = DocumentRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) // needed for legacy OCR-services + .features(features) + .build(); + + rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE, TenantContext.getTenantId(), request, message -> { + message.getMessageProperties().setPriority(priority); + return message; + }); } @@ -893,13 +909,13 @@ public class FileStatusService { websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE); if (!excluded) { - setStatusFullReprocess(dossierId, fileId, false, true); + setStatusFullReprocess(dossierId, fileId, false, true, false); } } @Transactional - public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) { + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -908,11 +924,17 @@ public class FileStatusService { return; } - if (requiresStructureAnalysis) { + if (requiresStructureAnalysis || runOcr) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId); } + if (runOcr) { + fileStatusPersistenceService.resetOcrStartAndEndDate(fileId); + setStatusOcrQueued(dossierId, fileId); + return; + } + addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT); } @@ -925,7 +947,7 @@ public class FileStatusService { websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE); if (!excludedFromAutomaticAnalysis) { - setStatusFullReprocess(dossierId, fileId, false, true); + setStatusFullReprocess(dossierId, fileId, false, true, false); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index a72ea2d19..47c6f2011 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -48,10 +48,10 @@ public class ReanalysisService { private final Predicate errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR); - public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis) { + public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis, boolean runOcr) { var errorFiles = fileStatusService.getAllRelevantErrorFiles(); - analyseFiles(repeatStructureAnalysis, errorFiles); + analyseFiles(repeatStructureAnalysis, errorFiles, runOcr); } @@ -88,7 +88,7 @@ public class ReanalysisService { if (force) { filesToReanalyse.forEach(file -> { if (file.isFullAnalysisRequired()) { - fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false); + fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false); } else { fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true); } @@ -98,7 +98,7 @@ public class ReanalysisService { .filter(FileModel::isReanalysisRequired) .forEach(file -> { if (file.isFullAnalysisRequired()) { - fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false); + fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, false, false); } else { fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, true); } @@ -107,9 +107,9 @@ public class ReanalysisService { } - private void analyseFiles(boolean repeatStructureAnalysis, List filesToReanalyse) { + private void analyseFiles(boolean repeatStructureAnalysis, List filesToReanalyse, boolean runOcr) { - filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis)); + filesToReanalyse.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), filesToReanalyse.size() == 1, repeatStructureAnalysis, runOcr)); } @@ -120,9 +120,9 @@ public class ReanalysisService { } - public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set fileIds, boolean repeatStructureAnalysis) { + public void reanalyzeGivenErrorFilesInDossier(String dossierId, Set fileIds, boolean repeatStructureAnalysis, boolean runOcr) { - analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds)); + analyseFiles(repeatStructureAnalysis, fileIds.isEmpty() ? getAllFilesForDossier(dossierId, errorFilesFilter) : getRelevantErrorFiles(dossierId, fileIds), runOcr); } @@ -231,7 +231,7 @@ public class ReanalysisService { } catch (Exception e) { throw new BadRequestException("File Not Valid PDF or number of pages do not match"); } - fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false); + fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false, false); } @@ -245,7 +245,7 @@ public class ReanalysisService { deleteImportedRedactionsRequest.getFileId(), FileType.IMPORTED_REDACTIONS, importedRedactions); - fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false); + fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false, false); } @@ -263,7 +263,7 @@ public class ReanalysisService { fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) { - fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false); + fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false, false); } } @@ -281,7 +281,7 @@ public class ReanalysisService { List rejectedFiles = filterInvalidFiles(files); - files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis())); + files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis(), reanalysisSettings.runOcr())); return rejectedFiles; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index b5546898d..dcdcca15b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -125,7 +125,7 @@ public class AutomaticAnalysisJob implements Job { if (file.isFullAnalysisRequired()) { log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getId()); - fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false); + fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false, false); } else if (file.isReanalysisRequired()) { log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getId()); fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 6f8f21c69..e60b57372 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -194,6 +194,11 @@ public class DossierPersistenceService { return dossierRepository.findByDossierTemplateId(dossierTemplateId); } + public List findAllActiveDossiersForDossierTemplateId(String dossierTemplateId) { + + return dossierRepository.findActiveByDossierTemplateId(dossierTemplateId); + } + public Set findAllDossierIdsForDossierTemplateId(String dossierTemplateId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index fb0de27ee..f4d3744d7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -414,6 +414,16 @@ public class FileStatusPersistenceService { .toList(); } + public List getActiveFilesForDossierTemplate(String dossierTemplateId) { + + return dossierService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId) + .stream() + .map(dossier -> fileRepository.findByDossierId(dossier.getId())) + .flatMap(List::stream) + .filter(f -> !f.isSoftOrHardDeleted()) + .toList(); + } + public List getStatusesAddedBefore(OffsetDateTime end) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index a844fc87a..3c0a390c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -115,4 +115,7 @@ public interface DossierRepository extends JpaRepository @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL") Optional findDossierNameById(@Param("dossierId") String dossierId); + @Query("select d from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null") + List findActiveByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId); + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java index 33cc448a7..56e5684a7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java @@ -65,35 +65,35 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest var dossier = dossierTesterAndProvider.provideTestDossier(); var file = fileTesterAndProvider.testAndProvideFile(dossier); - var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true)); + var e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false)); assertThat(e.status()).isEqualTo(409); setProcessingStatusToError(dossier, file); - supportClient.reanalyzeAllRelevantErrorFiles(true); + supportClient.reanalyzeAllRelevantErrorFiles(true, false); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); setProcessingStatusToError(dossier, file); - supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true); + supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(), true, false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); setProcessingStatusToError(dossier, file); - supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true); + supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); setProcessingStatusToError(dossier, file); setFileToStatusDeleted(file); - e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true)); + e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false)); assertThat(e.status()).isEqualTo(400); setFileToStatusHardDeleted(file); - e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true)); + e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true, false)); assertThat(e.status()).isEqualTo(400); } @@ -177,7 +177,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest dossierClient.deleteDossier(dossier1.getId()); - supportClient.reanalyzeAllRelevantErrorFiles(false); + supportClient.reanalyzeAllRelevantErrorFiles(false, false); FeignException feignException = assertThrows(FeignException.class, () -> fileClient.getFileStatus(dossier1.getId(), file1.getId())); assertEquals(feignException.status(), 404); FileStatus fileStatus = FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(file1.getId())); @@ -187,11 +187,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest setProcessingStatusToError(dossier2, file2); - supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true); + supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); - feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true)); + feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false)); assertEquals(feignException.status(), 404); setProcessingStatusToError(dossier2, file2); @@ -200,6 +200,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()), Collections.emptySet(), true, + false, new FileStatusFilter(null, null, true, true))); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); @@ -224,7 +225,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest dossierClient.archiveDossiers(Collections.singleton(dossier1.getId())); - supportClient.reanalyzeAllRelevantErrorFiles(false); + supportClient.reanalyzeAllRelevantErrorFiles(false, false); FileStatus fileStatus = fileClient.getFileStatus(dossier1.getId(), file1.getId()); assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); @@ -232,11 +233,11 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest setProcessingStatusToError(dossier2, file2); - supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true); + supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true, false); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); - FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true)); + FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true, false)); assertEquals(feignException.status(), 404); setProcessingStatusToError(dossier2, file2); @@ -245,6 +246,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()), Collections.emptySet(), true, + false, null)); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java index da05515b3..177207e15 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -9,6 +9,7 @@ public record ReanalysisSettings( @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds, @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds, @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis, + @Schema(description = "If set to true, ocr will be repeated and therefore also layout parsing and named entity recognition.", defaultValue = "false") boolean runOcr, @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter ) {