diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 75f9d9ea7..9a3d7da7f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -40,7 +40,7 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } api("com.knecon.fforesight:azure-ocr-service-api:0.24.0") - implementation("com.knecon.fforesight:llm-service-api:1.17.0") + implementation("com.knecon.fforesight:llm-service-api:1.35.0") api("com.knecon.fforesight:jobs-commons:0.10.0") api("com.iqser.red.commons:storage-commons:2.50.0") api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index 1a5640526..8b0900d69 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -154,6 +154,8 @@ public class DossierTemplateEntity { dossierTemplateCopy.removeWatermark = dossierTemplateEntity.removeWatermark; dossierTemplateCopy.downloadFileTypes = dossierTemplateEntity.downloadFileTypes; dossierTemplateCopy.layoutParsingType = dossierTemplateEntity.layoutParsingType; + dossierTemplateCopy.rotationCorrectionByDefault = dossierTemplateEntity.rotationCorrectionByDefault; + dossierTemplateCopy.idpByDefault = dossierTemplateEntity.idpByDefault; return dossierTemplateCopy; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 8f3d1dfa7..7b4b14a3c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -158,9 +158,9 @@ public class FileEntity { @Column(name = "ocr_end_time") private OffsetDateTime ocrEndTime; - private Integer usedPromptTokens; + private int usedPromptTokens; - private Integer usedCompletionTokens; + private int usedCompletionTokens; @Column private boolean hasAnnotationComments; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 347993705..a94b52cb9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -352,7 +352,6 @@ public class FileStatusService { return; } - boolean forceAnalysis = false; if (settings.isLlmNerServiceEnabled()) { boolean objectExists = fileManagementStorageService.objectExists(dossierId, fileId, FileType.LLM_NER_ENTITIES); @@ -386,7 +385,7 @@ public class FileStatusService { boolean reanalyse = fileModel.isReanalysisRequired() || analysisType.equals(AnalysisType.MANUAL_REDACTION_REANALYZE); MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); - if(analysisType == AnalysisType.FORCE_ANALYSE || forceAnalysis) { + if (analysisType == AnalysisType.FORCE_ANALYSE || forceAnalysis) { messageType = MessageType.ANALYSE; } @@ -837,7 +836,7 @@ public class FileStatusService { fileStatusPersistenceService.updateWorkflowStatus(fileId, newWorkflowStatus, false); - if(oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) { + if (oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) { fileStatusPersistenceService.clearLastDownload(fileId); } } @@ -980,6 +979,13 @@ public class FileStatusService { @Transactional public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr) { + setStatusFullReprocess(dossierId, fileId, priority, requiresStructureAnalysis, runOcr, false); + } + + + @Transactional + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis, boolean runOcr, boolean idp) { + FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); if (fileStatus.isExcluded()) { @@ -987,14 +993,14 @@ public class FileStatusService { return; } - if (requiresStructureAnalysis || runOcr) { + if (requiresStructureAnalysis || runOcr || idp) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId); } - if (runOcr) { + if (runOcr || idp) { fileStatusPersistenceService.resetOcrStartAndEndDate(fileId); - setStatusOcrQueued(dossierId, fileId); + setStatusOcrQueued(dossierId, fileId, idp); return; } @@ -1081,6 +1087,7 @@ public class FileStatusService { addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT); } + @Transactional public void setStatusForceAnalyse(String dossierId, String fileId, boolean priority) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index e515576e7..ad63f46a1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -291,7 +291,8 @@ public class ReanalysisService { file.getId(), false, reanalysisSettings.isRepeatStructureAnalysis(), - reanalysisSettings.runOcr())); + reanalysisSettings.isRunOcr(), + reanalysisSettings.isRunIdp())); return rejectedFiles; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 4b39bf1a7..1add5c029 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -413,7 +413,7 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.ocrStartTime = NULL, f.ocrEndTime = NULL, f.numberOfPagesToOCR = NULL, f.numberOfOCRedPages = NULL where f.id = :fileId") + @Query("update FileEntity f set f.ocrStartTime = NULL, f.ocrEndTime = NULL, f.numberOfPagesToOCR = NULL, f.numberOfOCRedPages = NULL, f.numberOfIdpPages = NULL where f.id = :fileId") void resetOcrStartAndEndDate(@Param("fileId") String fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 35ea64190..e198cf911 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -260,6 +260,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/159-cleanup-truncated-indices.yaml - include: - file: db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml + file: db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml - include: - file: db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml + file: db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml similarity index 76% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml index c668eb241..4bf6be136 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/159.0.0-add-usage-fields-to-file-for-idp-and-llm.yaml @@ -9,18 +9,16 @@ databaseChangeLog: - column: name: used_completion_tokens type: int - defaultValueNumeric: 0 + defaultValueNumeric: "0" constraints: nullable: false - column: name: used_prompt_tokens type: int - defaultValueNumeric: 0 + defaultValueNumeric: "0" constraints: nullable: false - column: name: number_of_idp_pages type: int - defaultValueNumeric: 0 - constraints: - nullable: false + defaultValueNumeric: "0" diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml similarity index 100% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152.0.0-add-idp-related-fields-to-dossier-template.yaml rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/160.0.0-add-idp-related-fields-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java index 18ceab6e9..a36f80b49 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java @@ -191,7 +191,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest @Test @SneakyThrows - public void testDeletedFileOverrides() throws IOException { + public void testDeletedFileOverrides() { var dossier = dossierTesterAndProvider.provideTestDossier(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index bc40b95e3..21dd6d13b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -149,7 +149,10 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true); - List systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList()); + List systemManagedTypes = types.getTypes() + .stream() + .filter(TypeValue::isSystemManaged) + .collect(Collectors.toList()); assertThat(systemManagedTypes.size()).isEqualTo(8); var allTemplates = dossierTemplateClient.getAllDossierTemplates(); @@ -284,17 +287,17 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { dictionaryClient.getDictionaryForType(type2.getType(), type2.getDossierTemplateId(), dossier.getId()); var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes(); - assertThat(allTypes - .stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()) - .size()).isEqualTo(4); + assertThat(allTypes.stream() + .filter(t -> !t.isSystemManaged()) + .count()).isEqualTo(4); var typesWithRankOfType1 = allTypes.stream() .filter(t -> t.getRank() == type.getRank()) - .collect(Collectors.toList()); + .toList(); assertThat(typesWithRankOfType1.size()).isEqualTo(2); var typesWithRankOfType2 = allTypes.stream() .filter(t -> t.getRank() == type2.getRank()) - .collect(Collectors.toList()); + .toList(); assertThat(typesWithRankOfType2.size()).isEqualTo(2); dictionaryClient.addEntry(createdType1.getType(), createdType1.getDossierTemplateId(), List.of("entry1", "entry2"), false, null, DictionaryEntryType.ENTRY); @@ -596,7 +599,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { .build()); // add new justifications - legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason","technicalReason")), dossierTemplate.getId()); + legalBasisClient.setLegalBasisMapping(List.of(new LegalBasis("nameAgain", "description", "reason", "technicalReason")), dossierTemplate.getId()); existingLegalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplate.getId()); assertThat(existingLegalBasis.size()).isEqualTo(1); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java index 56e5684a7..07ecc60d4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java @@ -201,6 +201,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest Collections.emptySet(), true, false, + false, new FileStatusFilter(null, null, true, true))); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); @@ -247,6 +248,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest Collections.emptySet(), true, false, + false, null)); loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId()); assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java index 0a0d4879a..df8760d72 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -10,21 +10,22 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; +@Getter @Builder @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public final class ReanalysisSettings { - @Getter @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds; - @Getter @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds; - @Getter @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis; - + @Schema(description = "If set to true, OCR will be repeated.", defaultValue = "false") + boolean runOcr; + @Schema(description = "If set to true, OCR with IDP will be repeated.", defaultValue = "false") + boolean runIdp; @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "null") FileStatusFilter fileStatusFilter;