diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java index 2858db5f3..22203fb67 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java @@ -305,6 +305,7 @@ public class DossierTemplateController implements DossierTemplateResource { .keepHiddenText(dossierTemplate.isKeepHiddenText()) .keepOverlappingObjects(dossierTemplate.isKeepOverlappingObjects()) .applyDictionaryUpdatesToAllDossiersByDefault(dossierTemplate.isApplyDictionaryUpdatesToAllDossiersByDefault()) + .ocrByDefault(dossierTemplate.isOcrByDefault()) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index cccddd75f..e75e90f7e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -78,6 +78,9 @@ public class DossierTemplateEntity { @Column(name = "apply_dictionary_updates_to_all_dossiers_by_default") private boolean applyDictionaryUpdatesToAllDossiersByDefault; + @Column(name = "ocr_by_default") + private boolean ocrByDefault; + @Builder.Default @Column(columnDefinition = "text", name = "download_file_types") @Convert(converter = JSONDownloadFileTypeConverter.class) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index b6a93ffc7..25c8ff541 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -175,7 +175,8 @@ public class FileStatusService { var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper()); reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - if (settings.isOcrByDefault() && fileModel.getOcrEndTime() == null) { + var dossierTemplate = dossier.getDossierTemplate(); + if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null) { log.debug("Add file: {} from dossier {} to OCR queue", fileId, dossierId); setStatusOcrQueued(dossierId, fileId); return; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java index 517faabac..452013c4b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java @@ -37,8 +37,6 @@ public class FileManagementServiceSettings { private boolean scmEnabled; - private boolean ocrByDefault; - private double compressionThresholdRatio = 10; private int tenantAccessTokenLifeSpan = 300; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index dc73b14ca..73cb71de7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -147,3 +147,5 @@ databaseChangeLog: file: db/changelog/tenant/52-add-file-status-error-timestamp.yaml - include: file: db/changelog/tenant/53-add-apply_dictionary_updates_to_all_dossiers_to_dossier_template.yaml + - include: + file: db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml new file mode 100644 index 000000000..9d629f8c8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml @@ -0,0 +1,12 @@ +databaseChangeLog: + - changeSet: + id: add-ocr-by-default-to-dossier-template + author: andreiisvoran + changes: + - addColumn: + columns: + - column: + name: ocr_by_default + type: BOOLEAN + defaultValue: false + tableName: dossier_template \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java index daa85981e..8b88edd60 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java @@ -44,18 +44,29 @@ public class DossierTemplateTesterAndProvider { public DossierTemplateModel provideTestTemplate() { - return provideTestTemplate("Template 1"); + return provideTestTemplate("Template 1", false); + } + + public DossierTemplateModel provideTestTemplate(boolean ocrByDefault) { + + return provideTestTemplate("Template 1", ocrByDefault); } public DossierTemplateModel provideTestTemplate(String name) { + return provideTestTemplate(name, false); + } + + public DossierTemplateModel provideTestTemplate(String name, boolean ocrByDefault) { + var cru = new DossierTemplateModel(); cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); cru.setName(name); cru.setDescription("Template 1"); cru.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); cru.setValidTo(OffsetDateTime.now().plusHours(1).truncatedTo(ChronoUnit.MILLIS)); + cru.setOcrByDefault(ocrByDefault); var result = dossierTemplateClient.createOrUpdateDossierTemplate(cru); @@ -70,7 +81,6 @@ public class DossierTemplateTesterAndProvider { loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId()); - return loadedTemplate; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 3da16c986..e58c984a1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -3,19 +3,20 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; import java.io.InputStream; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.mock.web.MockMultipartFile; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; @@ -41,6 +42,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkMo import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType; @@ -719,4 +721,30 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { // } } + @Test + public void testDossierTemplateWithOcrByDefault() { + + String name = "Test Dossier Template"; + + DossierTemplateModel dossierTemplateModel = new DossierTemplateModel(); + dossierTemplateModel.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); + dossierTemplateModel.setName(name); + dossierTemplateModel.setDescription("Template 1"); + dossierTemplateModel.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + dossierTemplateModel.setValidTo(OffsetDateTime.now().plusHours(1).truncatedTo(ChronoUnit.MILLIS)); + dossierTemplateModel.setOcrByDefault(true); + + DossierTemplateModel result = dossierTemplateClient.createOrUpdateDossierTemplate(dossierTemplateModel); + assertThat(result.getName()).isEqualTo(name); + assertThat(result.isOcrByDefault()).isTrue(); + + DossierTemplateModel loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId()); + assertThat(loadedTemplate).isEqualTo(result); + + dossierTemplateModel.setName("Test Dossier Template Update"); + dossierTemplateModel.setOcrByDefault(false); + loadedTemplate = dossierTemplateClient.createOrUpdateDossierTemplate(dossierTemplateModel); + assertThat(loadedTemplate.isOcrByDefault()).isFalse(); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index a1622b13b..fabc205e9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -2,6 +2,8 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -14,7 +16,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.mock.web.MockMultipartFile; import com.google.common.collect.Sets; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; +import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; @@ -30,13 +35,20 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange; import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequest; @@ -46,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Lega import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequest; import feign.FeignException; +import lombok.SneakyThrows; public class FileTest extends AbstractPersistenceServerServiceTest { @@ -94,6 +107,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Autowired private UserProvider userProvider; + @Autowired + private DossierTemplateClient dossierTemplateClient; + + @Autowired + private FileManagementStorageService fileManagementStorageService; + @Test public void testFileSoftDeleteReupload() { @@ -561,4 +580,25 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastReviewer()).isNull(); assertThat(loadedFile.getLastApprover()).isNull(); } + + @Test + @SneakyThrows + public void testUploadFileIsInOcrProcessingWithOcrByDefaultFlagTrue() { + + DossierTemplateModel dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(true); + Dossier dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "1"); + String fileName = "test"; + HashFunction hashFunction = Hashing.murmur3_128(); + String fileId = hashFunction.hashBytes(((fileName + ".pdf") + dossier.getId()).getBytes(StandardCharsets.UTF_8)).toString(); + + fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.ORIGIN, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8))); + MockMultipartFile mff = new MockMultipartFile(fileName, fileName + ".pdf", "application/pdf", "lorem ipsum".getBytes()); + FileUploadResult uploadResult = uploadClient.upload(mff, dossier.getId(), false); + + fileId = uploadResult.getFileIds().iterator().next(); + FileStatus file = fileClient.getFileStatus(dossier.getId(), fileId); + + assertThat(file.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); + } + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java index aee26d68f..6fec62e5f 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java @@ -65,6 +65,8 @@ public class DossierTemplateModel { @Schema(description = "Representing the setting if dictionary updates is applied to all dossiers") private boolean applyDictionaryUpdatesToAllDossiersByDefault; + @Schema(description = "Flag that specifies if OCR is automatically performed on upload for all dossiers of this template") + private boolean ocrByDefault; public String getId() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java index bd7baf76a..df9d169bc 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java @@ -37,4 +37,6 @@ public class CreateOrUpdateDossierTemplateRequest { private boolean applyDictionaryUpdatesToAllDossiersByDefault; + private boolean ocrByDefault; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java index f5eb77257..282f6d272 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java @@ -34,4 +34,6 @@ public class DossierTemplate { private boolean keepHiddenText; private boolean keepOverlappingObjects; private boolean applyDictionaryUpdatesToAllDossiersByDefault; + private boolean ocrByDefault; + }