From a03ab9f105561618f1f37857a99344b060b35eaf Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 12:59:00 +0100 Subject: [PATCH 01/10] RED-8670: add idp fields and llm tracking --- .../impl/controller/ReanalysisController.java | 16 ++++--- .../external/resource/ReanalysisResource.java | 10 ++-- .../build.gradle.kts | 11 +++-- .../entity/dossier/DossierTemplateEntity.java | 6 +++ .../processor/entity/dossier/FileEntity.java | 7 +++ .../model/OCRStatusUpdateResponse.java | 20 -------- .../FileStatusProcessingUpdateService.java | 7 ++- .../processor/service/FileStatusService.java | 37 ++++++++++++--- .../processor/service/ReanalysisService.java | 29 ++++++------ .../FileStatusPersistenceService.java | 13 +++++- .../repository/FileRepository.java | 13 +++++- .../service/queue/NerMessageReceiver.java | 2 +- .../queue/OCRProcessingMessageReceiver.java | 7 +-- .../db/changelog/db.changelog-tenant.yaml | 4 ++ ...-usage-fields-to-file-for-idp-and-llm.yaml | 26 +++++++++++ ...dp-related-fields-to-dossier-template.yaml | 46 +++++++++++++++++++ .../integration/tests/ReanalysisTest.java | 8 ++-- .../test/resources/testcontainers.properties | 1 - .../shared/model/DossierTemplateModel.java | 6 +++ .../v1/api/shared/model/FileStatus.java | 6 +++ .../v1/api/shared/model/FileStatusFilter.java | 15 +++--- .../api/shared/model/ReanalysisSettings.java | 32 +++++++++---- .../dossiertemplate/DossierTemplate.java | 2 + .../dossier/file/FileModel.java | 4 ++ 24 files changed, 247 insertions(+), 81 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml delete mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java index 25ba16d58..beed0e3e1 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java @@ -36,6 +36,7 @@ import lombok.RequiredArgsConstructor; public class ReanalysisController implements ReanalysisResource { private static final String DOSSIER_ID = "dossierId"; + private final ReanalysisService reanalysisService; private final FileStatusManagementService fileStatusManagementService; private final AuditPersistenceService auditPersistenceService; @@ -98,11 +99,11 @@ public class ReanalysisController implements ReanalysisResource { @Override @PreAuthorize("hasAuthority('" + REANALYZE_DOSSIER + "')") - public void ocrDossier(@PathVariable(DOSSIER_ID) String dossierId) { + public void ocrDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp) { accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); - reanalysisService.ocrDossier(dossierId); + reanalysisService.ocrDossier(dossierId, idp); auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -118,11 +119,12 @@ public class ReanalysisController implements ReanalysisResource { @PreAuthorize("hasAuthority('" + REANALYZE_FILE + "')") public void ocrFile(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) { + @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force, + @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp) { accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); validateOCR(dossierId, fileId); - reanalysisService.ocrFile(dossierId, fileId, force); + reanalysisService.ocrFile(dossierId, fileId, force, idp); auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(dossierId) @@ -136,11 +138,13 @@ public class ReanalysisController implements ReanalysisResource { @Override @PreAuthorize("hasAuthority('" + REANALYZE_FILE + "')") - public void ocrFiles(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody Set fileIds) { + public void ocrFiles(@PathVariable(DOSSIER_ID) String dossierId, + @RequestBody Set fileIds, + @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp) { accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); fileIds.forEach(fileId -> validateOCR(dossierId, fileId)); - reanalysisService.ocrFiles(dossierId, fileIds); + reanalysisService.ocrFiles(dossierId, fileIds, idp); auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(dossierId) diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java index 62ed4ae56..52092bcd5 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java @@ -38,6 +38,7 @@ public interface ReanalysisResource { String EXCLUDED_STATUS_PARAM = "excluded"; String FORCE_PARAM = "force"; + String IDP_PARAM = "idp"; @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE) @@ -65,7 +66,7 @@ public interface ReanalysisResource { @Operation(summary = "Ocr and reanalyze a dossier", description = "None") @PostMapping(value = OCR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE) @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")}) - void ocrDossier(@PathVariable(DOSSIER_ID) String dossierId); + void ocrDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp); @Operation(summary = "Ocr and reanalyze a file", description = "None") @@ -73,13 +74,16 @@ public interface ReanalysisResource { @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "409", description = "Conflict"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden"), @ApiResponse(responseCode = "400", description = "Cannot OCR approved file")}) void ocrFile(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force); + @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force, + @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp); @Operation(summary = "Ocr and reanalyze multiple files for a dossier", description = "None") @PostMapping(value = OCR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH) @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")}) - void ocrFiles(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody Set fileIds); + void ocrFiles(@PathVariable(DOSSIER_ID) String dossierId, + @RequestBody Set fileIds, + @RequestParam(value = IDP_PARAM, required = false, defaultValue = FALSE) boolean idp); @Operation(summary = "Exclude or re-include a file to the automatic analysis", description = "None") diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index a9640eec9..75f9d9ea7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -31,13 +31,17 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } + api("com.knecon.fforesight:layoutparser-service-internal-api:0.181.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } - api("com.knecon.fforesight:azure-ocr-service-api:0.13.0") - implementation("com.knecon.fforesight:llm-service-api:1.20.0-RED10072.2") - api("com.knecon.fforesight:jobs-commons:0.13.0") + api("com.knecon.fforesight:azure-ocr-service-api:0.24.0") + implementation("com.knecon.fforesight:llm-service-api:1.17.0") + api("com.knecon.fforesight:jobs-commons:0.10.0") api("com.iqser.red.commons:storage-commons:2.50.0") api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") { exclude(group = "com.iqser.red.commons", module = "storage-commons") @@ -71,6 +75,7 @@ dependencies { api("commons-validator:commons-validator:1.7") api("com.opencsv:opencsv:5.9") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.mapstruct:mapstruct:1.6.2") annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index 63ea4c1dc..1a5640526 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -83,6 +83,12 @@ public class DossierTemplateEntity { @Column(name = "ocr_by_default") private boolean ocrByDefault; + @Column(name = "rotation_correction_by_default") + private boolean rotationCorrectionByDefault; + + @Column(name = "idp_by_default") + private boolean idpByDefault; + @Column(name = "remove_watermark") private boolean removeWatermark; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 188f52130..8f3d1dfa7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -152,9 +152,16 @@ public class FileEntity { @Column(name = "number_of_ocred_pages") private Integer numberOfOCRedPages; + @Column(name = "number_of_idp_pages") + private Integer numberOfIdpPages; + @Column(name = "ocr_end_time") private OffsetDateTime ocrEndTime; + private Integer usedPromptTokens; + + private Integer usedCompletionTokens; + @Column private boolean hasAnnotationComments; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java deleted file mode 100644 index 41c0d88e4..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@Builder -public class OCRStatusUpdateResponse { - - private String fileId; - private int numberOfPagesToOCR; - private int numberOfOCRedPages; - private boolean ocrFinished; - private boolean ocrStarted; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 190b23550..0a7ef0e36 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.util.Set; + import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; @@ -15,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDo import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.search.v1.model.IndexMessageType; +import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; @@ -114,7 +117,7 @@ public class FileStatusProcessingUpdateService { } - public void requeueOCROrMarkFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { + public void requeueOCROrMarkFailed(String dossierId, String fileId, Set features, FileErrorInfo fileErrorInfo) { var fileEntity = fileStatusPersistenceService.getStatus(fileId); if (fileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()) { @@ -122,7 +125,7 @@ public class FileStatusProcessingUpdateService { } else { fileStatusService.setStatusOcrProcessing(fileId, fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); - fileStatusService.addToOcrQueue(dossierId, fileId, 2); + fileStatusService.addToOcrQueue(dossierId, fileId, 2, features); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 9284cc37e..347993705 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -12,7 +12,6 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; - import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; @@ -21,6 +20,7 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocu import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection; @@ -31,7 +31,6 @@ import com.iqser.red.service.persistence.management.v1.processor.model.AnalysisT import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier; import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; -import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.VisualLayoutParsingServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; @@ -80,6 +79,7 @@ import com.knecon.fforesight.llm.service.LlmNerMessage; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature; import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest; +import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse; import com.knecon.fforesight.tenantcommons.TenantContext; import jakarta.transaction.Transactional; @@ -569,6 +569,12 @@ public class FileStatusService { public void setStatusOcrQueued(String dossierId, String fileId) { + setStatusOcrQueued(dossierId, fileId, false); + } + + + public void setStatusOcrQueued(String dossierId, String fileId, boolean idp) { + FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); if (fileStatus.isExcluded()) { @@ -579,7 +585,7 @@ public class FileStatusService { updateOCRStartTime(fileId); fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING_QUEUED); websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.OCR_PROCESSING, fileStatus.getNumberOfAnalyses() + 1); - addToOcrQueue(dossierId, fileId, 2); + addToOcrQueue(dossierId, fileId, 2, idp); } @@ -760,18 +766,29 @@ public class FileStatusService { } - public void addToOcrQueue(String dossierId, String fileId, int priority) { + public void addToOcrQueue(String dossierId, String fileId, int priority, boolean useIdp) { - var removeWatermark = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)).isRemoveWatermark(); + DossierTemplateEntity dt = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)); Set features = new HashSet<>(); - if (removeWatermark) { + if (dt.isRemoveWatermark()) { features.add(AzureOcrFeature.REMOVE_WATERMARKS); } - if (currentApplicationTypeProvider.isDocuMine()) { + if (useIdp || dt.isIdpByDefault()) { + features.add(AzureOcrFeature.IDP); + } + if (dt.isRotationCorrectionByDefault()) { features.add(AzureOcrFeature.ROTATION_CORRECTION); + } + if (currentApplicationTypeProvider.isDocuMine()) { features.add(AzureOcrFeature.FONT_STYLE_DETECTION); } + addToOcrQueue(dossierId, fileId, priority, features); + } + + + public void addToOcrQueue(String dossierId, String fileId, int priority, Set features) { + var request = DocumentRequest.builder() .dossierId(dossierId) .fileId(fileId) @@ -1175,4 +1192,10 @@ public class FileStatusService { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModels); } + + public void increaseTokenUsage(String fileId, int promptTokens, int completionTokens) { + + fileStatusPersistenceService.increaseTokenUsage(fileId, promptTokens, completionTokens); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index d66358877..e515576e7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -171,18 +171,18 @@ public class ReanalysisService { } - public void ocrDossier(String dossierId) { + public void ocrDossier(String dossierId, boolean idp) { var relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter); relevantFiles.stream() .filter(fileStatus -> fileStatus.getOcrStartTime() == null) .filter(fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.PROCESSED)) - .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); + .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId(), idp)); } - public void ocrFile(String dossierId, String fileId, boolean force) { + public void ocrFile(String dossierId, String fileId, boolean force, boolean idp) { dossierPersistenceService.getAndValidateDossier(dossierId); FileModel dossierFile = fileStatusService.getStatus(fileId); @@ -202,30 +202,31 @@ public class ReanalysisService { } if (force) { - fileStatusService.setStatusOcrQueued(dossierId, fileId); + fileStatusService.setStatusOcrQueued(dossierId, fileId, idp); } else { if (dossierFile.getOcrStartTime() != null) { throw new ConflictException("File already has been OCR processed"); } - ocrFiles(dossierId, Sets.newHashSet(fileId)); + ocrFiles(dossierId, Sets.newHashSet(fileId), idp); } } - public void ocrFiles(String dossierId, Set fileIds) { + public void ocrFiles(String dossierId, Set fileIds, boolean idp) { var relevantFiles = getRelevantFiles(dossierId, fileIds); if (relevantFiles.stream() - .anyMatch(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.PROCESSED) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.OCR_PROCESSING_QUEUED) && !fileStatus.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING))) { + .anyMatch(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.PROCESSED) // + && !fileStatus.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING_QUEUED) // + && !fileStatus.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING))) { throw new ConflictException("File is not processed"); } relevantFiles.stream() .filter(fileStatus -> fileStatus.getOcrStartTime() == null) - .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); + .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId(), idp)); } @@ -275,12 +276,12 @@ public class ReanalysisService { public List reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { - requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds()); + requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.getDossierIds(), reanalysisSettings.getFileIds()); var files = fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() .filter(file -> isInList(file, reanalysisSettings)) - .filter(reanalysisSettings.fileStatusFilter().asPredicate()) + .filter(reanalysisSettings.getFileStatusFilter()) .peek(file -> log.info("Reanalyzing file {}", file.getId())) .collect(Collectors.toList()); @@ -289,7 +290,7 @@ public class ReanalysisService { files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, - reanalysisSettings.repeatStructureAnalysis(), + reanalysisSettings.isRepeatStructureAnalysis(), reanalysisSettings.runOcr())); return rejectedFiles; @@ -314,8 +315,8 @@ public class ReanalysisService { private boolean isInList(FileModel file, ReanalysisSettings reAnalysisSettings) { - return (reAnalysisSettings.fileIds().isEmpty() || reAnalysisSettings.fileIds().contains(file.getId())) // - && (reAnalysisSettings.dossierIds().isEmpty() || reAnalysisSettings.dossierIds().contains(file.getDossierId())); + return (reAnalysisSettings.getFileIds().isEmpty() || reAnalysisSettings.getFileIds().contains(file.getId())) // + && (reAnalysisSettings.getDossierIds().isEmpty() || reAnalysisSettings.getDossierIds().contains(file.getDossierId())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 7ab237e08..10daaa7d0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -10,6 +10,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection; + import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -21,7 +23,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.projecti import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier; -import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; @@ -31,6 +32,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.component.C import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature; +import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse; import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; @@ -680,6 +683,7 @@ public class FileStatusPersistenceService { fileRepository.updateOCRStatus(response.getFileId(), response.getNumberOfPagesToOCR(), response.getNumberOfOCRedPages(), + response.getFeatures().contains(AzureOcrFeature.IDP) ? response.getNumberOfOCRedPages() : 0, response.isOcrFinished() ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -765,4 +769,11 @@ public class FileStatusPersistenceService { fileRepository.updateLastDownloadForFile(fileId, null); } + + @Transactional + public void increaseTokenUsage(String fileId, int promptTokens, int completionTokens) { + + fileRepository.increaseTokenUsage(fileId, promptTokens, completionTokens); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 20735338c..4b39bf1a7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -389,11 +389,15 @@ public interface FileRepository extends JpaRepository { @Transactional @Modifying(clearAutomatically = true) @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, " - + "f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, " - + "f.lastUpdated = :lastUpdated where f.id = :fileId") + + "f.numberOfPagesToOCR = :numberOfPagesToOCR, " + + "f.numberOfIdpPages = :numberOfIdpPages, " + + "f.ocrEndTime = :ocrEndTime, " + + "f.lastUpdated = :lastUpdated " + + "where f.id = :fileId") void updateOCRStatus(@Param("fileId") String fileId, @Param("numberOfPagesToOCR") int numberOfPagesToOCR, @Param("numberOfOCRedPages") int numberOfOCRedPages, + @Param("numberOfIdpPages") int numberOfIdpPages, @Param("ocrEndTime") OffsetDateTime ocrEndTime, @Param("lastUpdated") OffsetDateTime lastUpdated); @@ -479,6 +483,11 @@ public interface FileRepository extends JpaRepository { @Query("SELECT f FROM FileEntity f WHERE f.id in :fileIds AND f.dossierId = :dossierId") List findAllDossierIdAndIds(@Param("dossierId") String dossierId, @Param("fileIds") Set fileIds); + + @Modifying + @Query("UPDATE FileEntity f SET f.usedPromptTokens = f.usedPromptTokens + :promptTokens, f.usedCompletionTokens = f.usedCompletionTokens + :completionTokens WHERE f.id = :id") + void increaseTokenUsage(@Param("id") String fileId, @Param("promptTokens") int promptTokens, @Param("completionTokens") int completionTokens); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java index 31d6aee89..e8821fb8d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java @@ -63,7 +63,7 @@ public class NerMessageReceiver { String dossierId = QueueMessageIdentifierService.parseDossierId(message.getIdentifier()); String fileId = QueueMessageIdentifierService.parseFileId(message.getIdentifier()); addFileIdToTrace(fileId); - + fileStatusService.increaseTokenUsage(fileId, message.getPromptTokens(), message.getCompletionTokens()); log.info("Received message from {} for dossierId {} and fileId {}", LLM_ENTITY_RESPONSE_LISTENER_ID, dossierId, fileId); fileStatusPersistenceService.setAiCreationVersion(fileId, message.getAiCreationVersion()); fileStatusService.setStatusForceAnalyse(dossierId, fileId, false); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java index eb4be9542..cf6a0840a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java @@ -11,14 +11,13 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; -import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; - import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest; +import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -43,6 +42,8 @@ public class OCRProcessingMessageReceiver { @RabbitListener(id = OCR_STATUS_UPDATE_LISTENER_ID) public void handleOCRStatusUpdateMessage(OCRStatusUpdateResponse response) { + log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE); + var fileModel = fileStatusService.getStatus(response.getFileId()); if (response.isOcrStarted()) { @@ -57,7 +58,6 @@ public class OCRProcessingMessageReceiver { response.getNumberOfOCRedPages()); } - log.debug("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE); } @@ -95,6 +95,7 @@ public class OCRProcessingMessageReceiver { timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); fileStatusProcessingUpdateService.requeueOCROrMarkFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), + ocrRequestMessage.getFeatures(), new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp)); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 4f53c42f9..35ea64190 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -259,3 +259,7 @@ databaseChangeLog: file: db/changelog/tenant/158-add-app-version-history-table-and-layout-parser-version-field-to-file.yaml - include: file: db/changelog/tenant/159-cleanup-truncated-indices.yaml + - include: + file: db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml + - include: + file: db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml new file mode 100644 index 000000000..c668eb241 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml @@ -0,0 +1,26 @@ +databaseChangeLog: + - changeSet: + id: add-llm-usage-fields-to-file + author: kilian + changes: + - addColumn: + tableName: file + columns: + - column: + name: used_completion_tokens + type: int + defaultValueNumeric: 0 + constraints: + nullable: false + - column: + name: used_prompt_tokens + type: int + defaultValueNumeric: 0 + constraints: + nullable: false + - column: + name: number_of_idp_pages + type: int + defaultValueNumeric: 0 + constraints: + nullable: false diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml new file mode 100644 index 000000000..f4e4d59b1 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml @@ -0,0 +1,46 @@ +databaseChangeLog: + - changeSet: + id: add-idp-related-fields-to-dossier-template + author: kilian + changes: + - addColumn: + tableName: dossier_template + columns: + - column: + name: idp_by_default + type: boolean + defaultValueBoolean: false + remarks: "Indicates if IDP is enabled by default" + - column: + name: rotation_correction_by_default + type: boolean + remarks: "Indicates if rotation correction is enabled by default" + + - update: + tableName: dossier_template + columns: + - column: + name: rotation_correction_by_default + valueBoolean: true + where: "layout_parsing_type = 'DOCUMINE_OLD'" + + - update: + tableName: dossier_template + columns: + - column: + name: rotation_correction_by_default + valueBoolean: false + where: "layout_parsing_type != 'DOCUMINE_OLD'" + + - changeSet: + id: make-fields-non-nullable + author: kilian + changes: + - addNotNullConstraint: + tableName: dossier_template + columnName: idp_by_default + columnDataType: boolean + - addNotNullConstraint: + tableName: dossier_template + columnName: rotation_correction_by_default + columnDataType: boolean diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java index 19f547b96..d223dba70 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java @@ -46,22 +46,22 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isNotEqualTo(ProcessingStatus.PROCESSED); - reanalysisClient.ocrDossier(dossier.getId()); + reanalysisClient.ocrDossier(dossier.getId(), false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isNotEqualTo(ProcessingStatus.PROCESSED); resetProcessingStatus(file); - reanalysisClient.ocrDossier(dossier.getId()); + reanalysisClient.ocrDossier(dossier.getId(), false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); - reanalysisClient.ocrFile(dossier.getId(), file.getId(), true); + reanalysisClient.ocrFile(dossier.getId(), file.getId(), true, false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); - reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId())); + reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId()), false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties b/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties deleted file mode 100644 index ee9bd06b8..000000000 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties +++ /dev/null @@ -1 +0,0 @@ -hub.image.name.prefix=docker-dev.knecon.com/tests/ \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java index b152d201b..42b2f8305 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java @@ -68,6 +68,12 @@ public class DossierTemplateModel { @Schema(description = "Flag that specifies if OCR is automatically performed on upload for all dossiers of this template") private boolean ocrByDefault; + @Schema(description = "Flag that specifies if rotation correction is attempted during OCR for all dossiers of this template") + private boolean rotationCorrectionByDefault; + + @Schema(description = "Flag that specifies if IDP is automatically performed on upload for all dossiers of this template") + private boolean idpByDefault; + @Schema(description = "Flag that specifies the watermark removal in documents will be performed before the OCR processing") private boolean removeWatermark; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 5598d25a2..d505208fc 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -88,6 +88,12 @@ public class FileStatus { private int numberOfPagesToOCR; @Schema(description = "Number of pages already OCRed by us") private int numberOfOCRedPages; + @Schema(description = "Number of pages already IDPed by us") + private int numberOfIdpPages; + @Schema(description = "Number of prompt tokens used by this file") + private int usedPromptTokens; + @Schema(description = "Number of completion tokens used by this file") + private int usedCompletionTokens; @Schema(description = "Shows if this file has been OCRed by us. End time of OCR Process") private OffsetDateTime ocrEndTime; @Schema(description = "Shows if this file has comments on annotations.") diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java index 7c2d24589..8067e02c7 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java @@ -15,7 +15,7 @@ import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor -public class FileStatusFilter { +public class FileStatusFilter implements Predicate { private List processingStatusList = new ArrayList<>(); private List workflowStatusList = new ArrayList<>(); @@ -32,7 +32,8 @@ public class FileStatusFilter { } - public Predicate asPredicate() { + @Override + public boolean test(FileModel fileModel) { if (this.getProcessingStatusList() == null) { this.setProcessingStatusList(new ArrayList<>()); @@ -42,10 +43,12 @@ public class FileStatusFilter { this.setWorkflowStatusList(new ArrayList<>()); } - return fileStatus -> (this.getProcessingStatusList().isEmpty() || this.getProcessingStatusList().contains(fileStatus.getProcessingStatus())) - && (this.getWorkflowStatusList().isEmpty() || this.getWorkflowStatusList().contains(fileStatus.getWorkflowStatus())) - && (this.isIncludeSoftDeletedFiles() || fileStatus.getDeleted() == null) - && (this.isIncludeHardDeletedFiles() || fileStatus.getHardDeletedTime() == null); + return (this.getProcessingStatusList().isEmpty() || this.getProcessingStatusList().contains(fileModel.getProcessingStatus())) + && (this.getWorkflowStatusList().isEmpty() + || this.getWorkflowStatusList() + .contains(fileModel.getWorkflowStatus())) + && (this.isIncludeSoftDeletedFiles() || fileModel.getDeleted() == null) + && (this.isIncludeHardDeletedFiles() || fileModel.getHardDeletedTime() == null); } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java index 177207e15..0a0d4879a 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -4,16 +4,32 @@ import java.util.Optional; import java.util.Set; import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; -public record ReanalysisSettings( - @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds, - @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds, - @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis, - @Schema(description = "If set to true, ocr will be repeated and therefore also layout parsing and named entity recognition.", defaultValue = "false") boolean runOcr, - @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter -) { +@Builder +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public final class ReanalysisSettings { - public FileStatusFilter fileStatusFilter() { + @Getter + @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") + Set dossierIds; + @Getter + @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") + Set fileIds; + @Getter + @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") + boolean repeatStructureAnalysis; + + @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "null") + FileStatusFilter fileStatusFilter; + + + public FileStatusFilter getFileStatusFilter() { return Optional.ofNullable(fileStatusFilter) .orElse(new FileStatusFilter()); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java index dd2ead36a..c38b8f67c 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java @@ -36,6 +36,8 @@ public class DossierTemplate { private boolean keepOverlappingObjects; private boolean applyDictionaryUpdatesToAllDossiersByDefault; private boolean ocrByDefault; + private boolean rotationCorrectionByDefault; + private boolean idpByDefault; private boolean removeWatermark; private LayoutParsingType layoutParsingType; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index bdadfabad..2d82005e0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -8,6 +8,7 @@ import java.util.Set; import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -60,6 +61,9 @@ public class FileModel { private OffsetDateTime ocrStartTime; private Integer numberOfPagesToOCR; private Integer numberOfOCRedPages; + private Integer numberOfIdpPages; + private int usedPromptTokens; + private int usedCompletionTokens; private OffsetDateTime ocrEndTime; private boolean hasAnnotationComments; private boolean excluded; -- 2.47.2 From ce3da37a8b073ab8082a5109f139a7bd23a6eec1 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 16:53:04 +0100 Subject: [PATCH 02/10] RED-8670: add idp fields and llm tracking --- .../build.gradle.kts | 2 +- .../entity/dossier/DossierTemplateEntity.java | 2 ++ .../processor/entity/dossier/FileEntity.java | 4 ++-- .../processor/service/FileStatusService.java | 19 +++++++++++++------ .../processor/service/ReanalysisService.java | 3 ++- .../repository/FileRepository.java | 2 +- .../db/changelog/db.changelog-tenant.yaml | 4 ++-- ...usage-fields-to-file-for-idp-and-llm.yaml} | 8 +++----- ...p-related-fields-to-dossier-template.yaml} | 0 .../tests/ComponentOverrideTest.java | 2 +- .../tests/DossierTemplateTest.java | 17 ++++++++++------- .../tests/SupportControllerTest.java | 2 ++ .../api/shared/model/ReanalysisSettings.java | 9 +++++---- 13 files changed, 44 insertions(+), 30 deletions(-) rename persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/{151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml => 159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml} (76%) rename persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/{152.0.0-add-idp-related-fields-to-dossier-template.yaml => 160.0.0-add-idp-related-fields-to-dossier-template.yaml} (100%) diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 75f9d9ea7..9a3d7da7f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -40,7 +40,7 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } api("com.knecon.fforesight:azure-ocr-service-api:0.24.0") - implementation("com.knecon.fforesight:llm-service-api:1.17.0") + implementation("com.knecon.fforesight:llm-service-api:1.35.0") api("com.knecon.fforesight:jobs-commons:0.10.0") api("com.iqser.red.commons:storage-commons:2.50.0") api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index 1a5640526..8b0900d69 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -154,6 +154,8 @@ public class DossierTemplateEntity { dossierTemplateCopy.removeWatermark = dossierTemplateEntity.removeWatermark; dossierTemplateCopy.downloadFileTypes = dossierTemplateEntity.downloadFileTypes; dossierTemplateCopy.layoutParsingType = dossierTemplateEntity.layoutParsingType; + dossierTemplateCopy.rotationCorrectionByDefault = dossierTemplateEntity.rotationCorrectionByDefault; + dossierTemplateCopy.idpByDefault = dossierTemplateEntity.idpByDefault; return dossierTemplateCopy; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 8f3d1dfa7..7b4b14a3c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -158,9 +158,9 @@ public class FileEntity { @Column(name = "ocr_end_time") private OffsetDateTime ocrEndTime; - private Integer usedPromptTokens; + private int usedPromptTokens; - private Integer usedCompletionTokens; + private int usedCompletionTokens; @Column private boolean hasAnnotationComments; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 347993705..a94b52cb9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -352,7 +352,6 @@ public class FileStatusService { return; } - boolean forceAnalysis = false; if (settings.isLlmNerServiceEnabled()) { boolean objectExists = fileManagementStorageService.objectExists(dossierId, fileId, FileType.LLM_NER_ENTITIES); @@ -386,7 +385,7 @@ public class FileStatusService { boolean reanalyse = fileModel.isReanalysisRequired() || analysisType.equals(AnalysisType.MANUAL_REDACTION_REANALYZE); MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); - if(analysisType == AnalysisType.FORCE_ANALYSE || forceAnalysis) { + if (analysisType == AnalysisType.FORCE_ANALYSE || forceAnalysis) { messageType = MessageType.ANALYSE; } @@ -837,7 +836,7 @@ public class FileStatusService { fileStatusPersistenceService.updateWorkflowStatus(fileId, newWorkflowStatus, false); - if(oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) { + if (oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) { fileStatusPersistenceService.clearLastDownload(fileId); } } @@ -980,6 +979,13 @@ public class FileStatusService { @Transactional public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr) { + setStatusFullReprocess(dossierId, fileId, priority, requiresStructureAnalysis, runOcr, false); + } + + + @Transactional + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr, boolean idp) { + FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); if (fileStatus.isExcluded()) { @@ -987,14 +993,14 @@ public class FileStatusService { return; } - if (requiresStructureAnalysis || runOcr) { + if (requiresStructureAnalysis || runOcr || idp) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId); } - if (runOcr) { + if (runOcr || idp) { fileStatusPersistenceService.resetOcrStartAndEndDate(fileId); - setStatusOcrQueued(dossierId, fileId); + setStatusOcrQueued(dossierId, fileId, idp); return; } @@ -1081,6 +1087,7 @@ public class FileStatusService { addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT); } + @Transactional public void setStatusForceAnalyse(String dossierId, String fileId, boolean priority) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index e515576e7..ad63f46a1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -291,7 +291,8 @@ public class ReanalysisService { file.getId(), false, reanalysisSettings.isRepeatStructureAnalysis(), - reanalysisSettings.runOcr())); + reanalysisSettings.isRunOcr(), + reanalysisSettings.isRunIdp())); return rejectedFiles; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 4b39bf1a7..1add5c029 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -413,7 +413,7 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.ocrStartTime = NULL, f.ocrEndTime = NULL, f.numberOfPagesToOCR = NULL, f.numberOfOCRedPages = NULL where f.id = :fileId") + @Query("update FileEntity f set f.ocrStartTime = NULL, f.ocrEndTime = NULL, f.numberOfPagesToOCR = NULL, f.numberOfOCRedPages = NULL, f.numberOfIdpPages = NULL where f.id = :fileId") void resetOcrStartAndEndDate(@Param("fileId") String fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 35ea64190..e198cf911 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -260,6 +260,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/159-cleanup-truncated-indices.yaml - include: - file: db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml + file: db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml - include: - file: db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml + file: db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml similarity index 76% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml index c668eb241..4bf6be136 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml @@ -9,18 +9,16 @@ databaseChangeLog: - column: name: used_completion_tokens type: int - defaultValueNumeric: 0 + defaultValueNumeric: "0" constraints: nullable: false - column: name: used_prompt_tokens type: int - defaultValueNumeric: 0 + defaultValueNumeric: "0" constraints: nullable: false - column: name: number_of_idp_pages type: int - defaultValueNumeric: 0 - constraints: - nullable: false + defaultValueNumeric: "0" diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml similarity index 100% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java index 18ceab6e9..a36f80b49 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java @@ -191,7 +191,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest @Test @SneakyThrows - public void testDeletedFileOverrides() throws IOException { + public void testDeletedFileOverrides() { var dossier = dossierTesterAndProvider.provideTestDossier(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index bc40b95e3..21dd6d13b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -149,7 +149,10 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true); - List systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList()); + List systemManagedTypes = types.getTypes() + .stream() + .filter(TypeValue::isSystemManaged) + .collect(Collectors.toList()); assertThat(systemManagedTypes.size()).isEqualTo(8); var allTemplates = dossierTemplateClient.getAllDossierTemplates(); @@ -284,17 +287,17 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { dictionaryClient.getDictionaryForType(type2.getType(), type2.getDossierTemplateId(), dossier.getId()); var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes(); - assertThat(allTypes - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(4); + assertThat(allTypes.stream() + .filter(t -> !t.isSystemManaged()) + .count()).isEqualTo(4); var typesWithRankOfType1 = allTypes.stream() .filter(t -> t.getRank() == type.getRank()) - .collect(Collectors.toList()); + .toList(); assertThat(typesWithRankOfType1.size()).isEqualTo(2); var typesWithRankOfType2 = allTypes.stream() .filter(t -> t.getRank() == type2.getRank()) - .collect(Collectors.toList()); + .toList(); assertThat(typesWithRankOfType2.size()).isEqualTo(2); dictionaryClient.addEntry(createdType1.getType(), createdType1.getDossierTemplateId(), List.of("entry1", "entry2"), false, null, DictionaryEntryType.ENTRY); @@ -596,7 +599,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { .build()); // add new justifications - legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason","technicalReason")), dossierTemplate.getId()); + legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason", "technicalReason")), dossierTemplate.getId()); existingLegalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplate.getId()); assertThat(existingLegalBasis.size()).isEqualTo(1); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java index 56e5684a7..07ecc60d4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java @@ -201,6 +201,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest Collections.emptySet(), true, false, + false, new FileStatusFilter(null, null, true, true))); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); @@ -247,6 +248,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest Collections.emptySet(), true, false, + false, null)); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java index 0a0d4879a..df8760d72 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -10,21 +10,22 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; +@Getter @Builder @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public final class ReanalysisSettings { - @Getter @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds; - @Getter @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds; - @Getter @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis; - + @Schema(description = "If set to true, OCR will be repeated.", defaultValue = "false") + boolean runOcr; + @Schema(description = "If set to true, OCR with IDP will be repeated.", defaultValue = "false") + boolean runIdp; @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "null") FileStatusFilter fileStatusFilter; -- 2.47.2 From a24cfd4b1850403ac8d21d60f2f1a79df9c64e22 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 16:54:45 +0100 Subject: [PATCH 03/10] RED-8670: add idp fields and llm tracking --- .../persistence-service-processor-v1/build.gradle.kts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 9a3d7da7f..5ec2f0f68 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -31,17 +31,13 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } - api("com.knecon.fforesight:layoutparser-service-internal-api:0.181.0") { - exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") - exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") - } api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } api("com.knecon.fforesight:azure-ocr-service-api:0.24.0") implementation("com.knecon.fforesight:llm-service-api:1.35.0") - api("com.knecon.fforesight:jobs-commons:0.10.0") + api("com.knecon.fforesight:jobs-commons:0.13.0") api("com.iqser.red.commons:storage-commons:2.50.0") api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") { exclude(group = "com.iqser.red.commons", module = "storage-commons") @@ -75,7 +71,6 @@ dependencies { api("commons-validator:commons-validator:1.7") api("com.opencsv:opencsv:5.9") - implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.mapstruct:mapstruct:1.6.2") annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2") -- 2.47.2 From 6656c9eb8a312005787a45743a1d44c29dd1af43 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 17:22:58 +0100 Subject: [PATCH 04/10] RED-8670: add idp fields and llm tracking --- .../dossiertemplate/CreateOrUpdateDossierTemplateRequest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java index abbc175f9..97f354ab8 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java @@ -41,6 +41,10 @@ public class CreateOrUpdateDossierTemplateRequest { private boolean ocrByDefault; + private boolean idpByDefault; + + private boolean rotationCorrectionByDefault; + private boolean removeWatermark; private LayoutParsingType layoutParsingType; -- 2.47.2 From 758aa3b1656cb2ab87e437f4c8e6f662a0c79b4b Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 17:49:26 +0100 Subject: [PATCH 05/10] RED-8670: add idp fields and llm tracking --- .../controller/DossierTemplateController.java | 2 ++ .../processor/service/FileStatusService.java | 6 ++-- .../tests/DossierTemplateTest.java | 30 +++++++++++++++++++ .../shared/model/DossierTemplateModel.java | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java index 1e9baa3c9..5b145cf10 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java @@ -314,6 +314,8 @@ public class DossierTemplateController implements DossierTemplateResource { .applyDictionaryUpdatesToAllDossiersByDefault(dossierTemplate.isApplyDictionaryUpdatesToAllDossiersByDefault()) .ocrByDefault(dossierTemplate.isOcrByDefault()) .removeWatermark(dossierTemplate.isRemoveWatermark()) + .idpByDefault(dossierTemplate.isIdpByDefault()) + .rotationCorrectionByDefault(dossierTemplate.isRotationCorrectionByDefault()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index a94b52cb9..4029891b8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -769,6 +769,9 @@ public class FileStatusService { DossierTemplateEntity dt = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)); Set features = new HashSet<>(); + + features.add(AzureOcrFeature.FONT_STYLE_DETECTION); + if (dt.isRemoveWatermark()) { features.add(AzureOcrFeature.REMOVE_WATERMARKS); } @@ -778,9 +781,6 @@ public class FileStatusService { if (dt.isRotationCorrectionByDefault()) { features.add(AzureOcrFeature.ROTATION_CORRECTION); } - if (currentApplicationTypeProvider.isDocuMine()) { - features.add(AzureOcrFeature.FONT_STYLE_DETECTION); - } addToOcrQueue(dossierId, fileId, priority, features); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 21dd6d13b..06467205f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -1,6 +1,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -1005,4 +1006,33 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { assertTrue(exception.getMessage().contains("Invalid dates! validFrom can't be after validTo.")); } + @Test + public void testUpdateDossierTemplateWithIdp() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var allTemplates = dossierTemplateClient.getAllDossierTemplates(); + assertThat(allTemplates.size()).isEqualTo(1); + assertThat(allTemplates.get(0)).isEqualTo(dossierTemplate); + + // update + var cru = new DossierTemplateModel(); + cru.setDossierTemplateId(dossierTemplate.getId()); + BeanUtils.copyProperties(dossierTemplate, cru); + String updatedName = "Template 1 Update"; + cru.setName(updatedName); + cru.setIdpByDefault(true); + cru.setRotationCorrectionByDefault(true); + + var updatedDT = dossierTemplateClient.createOrUpdateDossierTemplate(cru); + assertEquals(updatedName, updatedDT.getName()); + assertTrue(updatedDT.isIdpByDefault()); + assertTrue(updatedDT.isRotationCorrectionByDefault()); + + var loadedDT = dossierTemplateClient.getDossierTemplate(updatedDT.getId()); + assertEquals(updatedName, loadedDT.getName()); + assertTrue(loadedDT.isIdpByDefault()); + assertTrue(loadedDT.isRotationCorrectionByDefault()); + } + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java index 42b2f8305..399c43fb5 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java @@ -71,7 +71,7 @@ public class DossierTemplateModel { @Schema(description = "Flag that specifies if rotation correction is attempted during OCR for all dossiers of this template") private boolean rotationCorrectionByDefault; - @Schema(description = "Flag that specifies if IDP is automatically performed on upload for all dossiers of this template") + @Schema(description = "Flag that specifies if IDP is performed instead of OCR for all dossiers of this template") private boolean idpByDefault; @Schema(description = "Flag that specifies the watermark removal in documents will be performed before the OCR processing") -- 2.47.2 From 2a79f8b9953234b2df91d063d8157e5fd8c1c23f Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 17 Dec 2024 17:50:34 +0100 Subject: [PATCH 06/10] RED-8670: add idp fields and llm tracking --- .../service/persistence/FileStatusPersistenceService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 10daaa7d0..d412833a2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -10,8 +10,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection; - import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -- 2.47.2 From 8e4fde68305d21d9c40192df59faf089eb6caae4 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 18 Dec 2024 12:55:50 +0100 Subject: [PATCH 07/10] RED-8670: add idp fields and llm tracking --- .../api/impl/controller/DossierTemplateController.java | 1 + .../processor/entity/dossier/DossierTemplateEntity.java | 5 +++++ .../v1/processor/service/FileStatusService.java | 4 +++- .../persistence/FileStatusPersistenceService.java | 6 +++--- ...0.0.0-add-idp-related-fields-to-dossier-template.yaml | 9 +++++++++ .../v1/api/shared/model/DossierTemplateModel.java | 3 +++ .../CreateOrUpdateDossierTemplateRequest.java | 2 ++ .../shared/model/dossiertemplate/DossierTemplate.java | 1 + 8 files changed, 27 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java index 5b145cf10..91f1a6ab6 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java @@ -316,6 +316,7 @@ public class DossierTemplateController implements DossierTemplateResource { .removeWatermark(dossierTemplate.isRemoveWatermark()) .idpByDefault(dossierTemplate.isIdpByDefault()) .rotationCorrectionByDefault(dossierTemplate.isRotationCorrectionByDefault()) + .fontStyleDetection(dossierTemplate.isFontStyleDetection()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index 8b0900d69..4fa89592c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -89,6 +89,9 @@ public class DossierTemplateEntity { @Column(name = "idp_by_default") private boolean idpByDefault; + @Column(name = "font_style_detection") + private boolean fontStyleDetection; + @Column(name = "remove_watermark") private boolean removeWatermark; @@ -134,6 +137,7 @@ public class DossierTemplateEntity { @Enumerated(EnumType.STRING) private LayoutParsingType layoutParsingType; + public static DossierTemplateEntity copyDossierTemplateEntityWithoutChildEntities(DossierTemplateEntity dossierTemplateEntity) { DossierTemplateEntity dossierTemplateCopy = new DossierTemplateEntity(); @@ -156,6 +160,7 @@ public class DossierTemplateEntity { dossierTemplateCopy.layoutParsingType = dossierTemplateEntity.layoutParsingType; dossierTemplateCopy.rotationCorrectionByDefault = dossierTemplateEntity.rotationCorrectionByDefault; dossierTemplateCopy.idpByDefault = dossierTemplateEntity.idpByDefault; + dossierTemplateCopy.fontStyleDetection = dossierTemplateEntity.fontStyleDetection; return dossierTemplateCopy; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 4029891b8..600109a7f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -770,7 +770,9 @@ public class FileStatusService { DossierTemplateEntity dt = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)); Set features = new HashSet<>(); - features.add(AzureOcrFeature.FONT_STYLE_DETECTION); + if (dt.isFontStyleDetection()) { + features.add(AzureOcrFeature.FONT_STYLE_DETECTION); + } if (dt.isRemoveWatermark()) { features.add(AzureOcrFeature.REMOVE_WATERMARKS); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index d412833a2..fe5a7ef74 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -603,9 +603,9 @@ public class FileStatusPersistenceService { public int getNumberOfAssignedFiles(String userId) { List files = fileRepository.findFilesByAssignee(userId); - return files.stream() - .filter(fileEntity -> fileEntity.getHardDeletedTime() == null) - .collect(Collectors.toList()).size(); + return Math.toIntExact(files.stream() + .filter(fileEntity -> fileEntity.getHardDeletedTime() == null) + .count()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml index f4e4d59b1..3161a6c0a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml @@ -15,6 +15,11 @@ databaseChangeLog: name: rotation_correction_by_default type: boolean remarks: "Indicates if rotation correction is enabled by default" + - column: + name: font_style_detection + type: boolean + defaultValueBoolean: true + remarks: "Indicates if bold detection is enabled in OCR" - update: tableName: dossier_template @@ -44,3 +49,7 @@ databaseChangeLog: tableName: dossier_template columnName: rotation_correction_by_default columnDataType: boolean + - addNotNullConstraint: + tableName: dossier_template + columnName: font_style_detection + columnDataType: boolean diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java index 399c43fb5..b7df60e28 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java @@ -74,6 +74,9 @@ public class DossierTemplateModel { @Schema(description = "Flag that specifies if IDP is performed instead of OCR for all dossiers of this template") private boolean idpByDefault; + @Schema(description = "Flag that specifies if font style detection is performed during OCR") + private boolean fontStyleDetection; + @Schema(description = "Flag that specifies the watermark removal in documents will be performed before the OCR processing") private boolean removeWatermark; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java index 97f354ab8..0ed496b51 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java @@ -45,6 +45,8 @@ public class CreateOrUpdateDossierTemplateRequest { private boolean rotationCorrectionByDefault; + private boolean fontStyleDetection; + private boolean removeWatermark; private LayoutParsingType layoutParsingType; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java index c38b8f67c..91dff5de0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java @@ -39,6 +39,7 @@ public class DossierTemplate { private boolean rotationCorrectionByDefault; private boolean idpByDefault; private boolean removeWatermark; + private boolean fontStyleDetection; private LayoutParsingType layoutParsingType; } -- 2.47.2 From 8109e9f61b35d8ca37cc186796536e0da08c7d70 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 19 Dec 2024 11:14:36 +0100 Subject: [PATCH 08/10] RED-8670: add idp fields and llm tracking --- .../controller/DossierTemplateController.java | 5 ++--- .../entity/dossier/DossierTemplateEntity.java | 4 ++++ .../v1/processor/service/FileStatusService.java | 8 ++++++-- .../LayoutParsingRequestFactory.java | 7 +++++-- ...-idp-related-fields-to-dossier-template.yaml | 11 ++++++++++- .../integration/tests/DossierTemplateTest.java | 17 ++++++++++++----- .../build.gradle.kts | 2 +- .../api/shared/model/DossierTemplateModel.java | 3 +++ .../CreateOrUpdateDossierTemplateRequest.java | 2 ++ .../model/dossiertemplate/DossierTemplate.java | 1 + .../dossiertemplate/dossier/file/FileType.java | 1 + 11 files changed, 47 insertions(+), 14 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java index 91f1a6ab6..f5d6bb9c5 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java @@ -24,13 +24,13 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ExportDownloadRequest; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateStatsService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; import com.iqser.red.service.persistence.service.v1.api.external.resource.DossierTemplateResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; @@ -42,7 +42,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStats; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ExportDownloadRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; @@ -61,7 +60,6 @@ public class DossierTemplateController implements DossierTemplateResource { private final AuditPersistenceService auditPersistenceService; private final DossierManagementService dossierManagementService; private final DossierACLService dossierACLService; - private final UserService userService; @Override @@ -317,6 +315,7 @@ public class DossierTemplateController implements DossierTemplateResource { .idpByDefault(dossierTemplate.isIdpByDefault()) .rotationCorrectionByDefault(dossierTemplate.isRotationCorrectionByDefault()) .fontStyleDetection(dossierTemplate.isFontStyleDetection()) + .ocrAllPages(dossierTemplate.isOcrAllPages()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index 4fa89592c..fdd48e1ef 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -92,6 +92,9 @@ public class DossierTemplateEntity { @Column(name = "font_style_detection") private boolean fontStyleDetection; + @Column(name = "ocr_all_pages") + private boolean ocrAllPages; + @Column(name = "remove_watermark") private boolean removeWatermark; @@ -161,6 +164,7 @@ public class DossierTemplateEntity { dossierTemplateCopy.rotationCorrectionByDefault = dossierTemplateEntity.rotationCorrectionByDefault; dossierTemplateCopy.idpByDefault = dossierTemplateEntity.idpByDefault; dossierTemplateCopy.fontStyleDetection = dossierTemplateEntity.fontStyleDetection; + dossierTemplateCopy.ocrAllPages = dossierTemplateEntity.ocrAllPages; return dossierTemplateCopy; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 600109a7f..5592c1b05 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -773,7 +773,6 @@ public class FileStatusService { if (dt.isFontStyleDetection()) { features.add(AzureOcrFeature.FONT_STYLE_DETECTION); } - if (dt.isRemoveWatermark()) { features.add(AzureOcrFeature.REMOVE_WATERMARKS); } @@ -791,9 +790,14 @@ public class FileStatusService { public void addToOcrQueue(String dossierId, String fileId, int priority, Set features) { var request = DocumentRequest.builder() + // needed for legacy OCR-services .dossierId(dossierId) .fileId(fileId) - .removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) // needed for legacy OCR-services + .removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) + // new api + .originDocumentId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) + .viewerDocId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT)) + .idpResultId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IDP_RESULT)) .features(features) .build(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index fc2a5a94e..61f8d2ced 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -27,8 +27,7 @@ public class LayoutParsingRequestFactory { public LayoutParsingRequest build(String dossierTemplateId, String dossierId, String fileId, boolean priority) { - LayoutParsingType layoutParsingType = dossierTemplatePersistenceService.getDossierTemplate( - dossierTemplateId).getLayoutParsingType(); + LayoutParsingType layoutParsingType = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getLayoutParsingType(); Optional optionalImageFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO) // ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)) : Optional.empty(); @@ -39,6 +38,9 @@ public class LayoutParsingRequestFactory { Optional optionalVisualLayoutParsingFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.VISUAL_LAYOUT) // ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.VISUAL_LAYOUT)) : Optional.empty(); + Optional optionalIdpResultFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IDP_RESULT) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IDP_RESULT)) : Optional.empty(); + return LayoutParsingRequest.builder() .layoutParsingType(layoutParsingType) .identifier(QueueMessageIdentifierService.buildIdentifier(dossierId, fileId, priority)) @@ -55,6 +57,7 @@ public class LayoutParsingRequestFactory { .documentMarkdownFileStorageId(fileManagementServiceSettings.isStoreMarkdown() ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.MARKDOWN)) : Optional.empty()) + .idpResultStorageId(optionalIdpResultFileId) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml index 3161a6c0a..e583d541e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml @@ -19,7 +19,12 @@ databaseChangeLog: name: font_style_detection type: boolean defaultValueBoolean: true - remarks: "Indicates if bold detection is enabled in OCR" + remarks: "Indicates if font style detection is enabled in OCR" + - column: + name: ocr_all_pages + type: boolean + defaultValueBoolean: false + remarks: "Indicates if all pages should be processed during OCR instead of only pages with images" - update: tableName: dossier_template @@ -53,3 +58,7 @@ databaseChangeLog: tableName: dossier_template columnName: font_style_detection columnDataType: boolean + - addNotNullConstraint: + tableName: dossier_template + columnName: ocr_all_pages + columnDataType: boolean diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 06467205f..30ac173f8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -606,7 +606,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { // update dossier template metadata var cru = new DossierTemplateModel(); - cru.setDossierTemplateId(dossierTemplate.getId()); + cru.setId(dossierTemplate.getId()); BeanUtils.copyProperties(dossierTemplate, cru); cru.setName("Template 1 Update"); cru.setDescription("new description"); @@ -948,7 +948,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { assertThat(result.getName()).isEqualTo(name); assertThat(result.isOcrByDefault()).isTrue(); - DossierTemplateModel loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId()); + DossierTemplateModel loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getId()); assertThat(loadedTemplate).isEqualTo(result); dossierTemplateModel.setName("Test Dossier Template Update"); @@ -996,7 +996,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { // update var cru = new DossierTemplateModel(); - cru.setDossierTemplateId(dossierTemplate.getId()); + cru.setId(dossierTemplate.getId()); BeanUtils.copyProperties(dossierTemplate, cru); cru.setName("Template 1 Update"); cru.setValidTo(OffsetDateTime.of(2020, 1, 1, 1, 1, 1, 1, ZoneOffset.UTC)); @@ -1006,8 +1006,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { assertTrue(exception.getMessage().contains("Invalid dates! validFrom can't be after validTo.")); } + @Test - public void testUpdateDossierTemplateWithIdp() { + public void testUpdateDossierTemplateWithOCRSettings() { var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); @@ -1017,22 +1018,28 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { // update var cru = new DossierTemplateModel(); - cru.setDossierTemplateId(dossierTemplate.getId()); + cru.setId(dossierTemplate.getId()); BeanUtils.copyProperties(dossierTemplate, cru); String updatedName = "Template 1 Update"; cru.setName(updatedName); cru.setIdpByDefault(true); cru.setRotationCorrectionByDefault(true); + cru.setOcrAllPages(true); + cru.setFontStyleDetection(true); var updatedDT = dossierTemplateClient.createOrUpdateDossierTemplate(cru); assertEquals(updatedName, updatedDT.getName()); assertTrue(updatedDT.isIdpByDefault()); assertTrue(updatedDT.isRotationCorrectionByDefault()); + assertTrue(updatedDT.isFontStyleDetection()); + assertTrue(updatedDT.isOcrAllPages()); var loadedDT = dossierTemplateClient.getDossierTemplate(updatedDT.getId()); assertEquals(updatedName, loadedDT.getName()); assertTrue(loadedDT.isIdpByDefault()); assertTrue(loadedDT.isRotationCorrectionByDefault()); + assertTrue(loadedDT.isFontStyleDetection()); + assertTrue(loadedDT.isOcrAllPages()); } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts index 3b65aa3cc..08014c970 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { api("com.knecon.fforesight:document:${rootProject.extra.get("documentVersion")}"){ exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - api("com.knecon.fforesight:layoutparser-service-internal-api:0.194.0-RED9998.1") { + api("com.knecon.fforesight:layoutparser-service-internal-api:idp1") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java index b7df60e28..da03244ac 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java @@ -77,6 +77,9 @@ public class DossierTemplateModel { @Schema(description = "Flag that specifies if font style detection is performed during OCR") private boolean fontStyleDetection; + @Schema(description = "Flag that specifies if OCR should be performed on all pages instead of only pages with images") + private boolean ocrAllPages; + @Schema(description = "Flag that specifies the watermark removal in documents will be performed before the OCR processing") private boolean removeWatermark; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java index 0ed496b51..f2e1d9779 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java @@ -49,6 +49,8 @@ public class CreateOrUpdateDossierTemplateRequest { private boolean removeWatermark; + private boolean ocrAllPages; + private LayoutParsingType layoutParsingType; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java index 91dff5de0..a1b906263 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java @@ -40,6 +40,7 @@ public class DossierTemplate { private boolean idpByDefault; private boolean removeWatermark; private boolean fontStyleDetection; + private boolean ocrAllPages; private LayoutParsingType layoutParsingType; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index bb00f3608..d853cf0df 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -21,6 +21,7 @@ public enum FileType { TABLES(".json"), VISUAL_LAYOUT(".json"), + IDP_RESULT(".json"), COMPONENTS(".json"), // document is split into 4 files, all should be overridden/deleted at the same time DOCUMENT_TEXT_OLD(".json"), -- 2.47.2 From fb6a16dfe965c4de11ec9143649badd740d42adc Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 13 Jan 2025 13:32:32 +0100 Subject: [PATCH 09/10] RED-8670: make ocr settings configurable --- .../persistence-service-shared-api-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts index 08014c970..445647608 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { api("com.knecon.fforesight:document:${rootProject.extra.get("documentVersion")}"){ exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - api("com.knecon.fforesight:layoutparser-service-internal-api:idp1") { + api("com.knecon.fforesight:layoutparser-service-internal-api:0.196.0-RED8670.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } -- 2.47.2 From 599a57b7fde5781a260780c69f70e618f3c8dc08 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 14 Jan 2025 13:12:05 +0100 Subject: [PATCH 10/10] RED-8670: make ocr settings configurable --- .../src/main/resources/db/changelog/db.changelog-tenant.yaml | 4 ++-- ...yaml => 160-add-usage-fields-to-file-for-idp-and-llm.yaml} | 0 ...ml => 161-add-idp-related-fields-to-dossier-template.yaml} | 0 .../src/test/resources/testcontainers.properties | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) rename persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/{159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml => 160-add-usage-fields-to-file-for-idp-and-llm.yaml} (100%) rename persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/{160.0.0-add-idp-related-fields-to-dossier-template.yaml => 161-add-idp-related-fields-to-dossier-template.yaml} (100%) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index e198cf911..f2ad84fbd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -260,6 +260,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/159-cleanup-truncated-indices.yaml - include: - file: db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml + file: db/changelog/tenant/160-add-usage-fields-to-file-for-idp-and-llm.yaml - include: - file: db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml + file: db/changelog/tenant/161-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160-add-usage-fields-to-file-for-idp-and-llm.yaml similarity index 100% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160-add-usage-fields-to-file-for-idp-and-llm.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/161-add-idp-related-fields-to-dossier-template.yaml similarity index 100% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/161-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties b/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties new file mode 100644 index 000000000..ee9bd06b8 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties @@ -0,0 +1 @@ +hub.image.name.prefix=docker-dev.knecon.com/tests/ \ No newline at end of file -- 2.47.2