Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6992 - Add ocrByDefault flag for dossier template #58

Merged
andrei.isvoran.ext merged 1 commits from RED-6992-ocrByDefault into master 2023-07-28 10:29:17 +02:00
Conversation 6 Commits 1 Files Changed 12 +108 -7
12 changed files with 108 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java vendored
View File

@ -305,6 +305,7 @@ public class DossierTemplateController implements DossierTemplateResource {
.keepHiddenText(dossierTemplate.isKeepHiddenText())
.keepOverlappingObjects(dossierTemplate.isKeepOverlappingObjects())
.applyDictionaryUpdatesToAllDossiersByDefault(dossierTemplate.isApplyDictionaryUpdatesToAllDossiersByDefault())
.ocrByDefault(dossierTemplate.isOcrByDefault())
.build();
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java
View File

@ -78,6 +78,9 @@ public class DossierTemplateEntity {
@Column(name = "apply_dictionary_updates_to_all_dossiers_by_default")
private boolean applyDictionaryUpdatesToAllDossiersByDefault;
@Column(name = "ocr_by_default")
private boolean ocrByDefault;
@Builder.Default
@Column(columnDefinition = "text", name = "download_file_types")
@Convert(converter = JSONDownloadFileTypeConverter.class)

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -175,7 +175,8 @@ public class FileStatusService {
var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper());
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
if (settings.isOcrByDefault() && fileModel.getOcrEndTime() == null) {
var dossierTemplate = dossier.getDossierTemplate();
if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null) {
log.debug("Add file: {} from dossier {} to OCR queue", fileId, dossierId);
setStatusOcrQueued(dossierId, fileId);
return;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java
View File

@ -37,8 +37,6 @@ public class FileManagementServiceSettings {
private boolean scmEnabled;
private boolean ocrByDefault;
private double compressionThresholdRatio = 10;
private int tenantAccessTokenLifeSpan = 300;

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -147,3 +147,5 @@ databaseChangeLog:
file: db/changelog/tenant/52-add-file-status-error-timestamp.yaml
- include:
file: db/changelog/tenant/53-add-apply_dictionary_updates_to_all_dossiers_to_dossier_template.yaml
- include:
file: db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml

12
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml Normal file
View File

@ -0,0 +1,12 @@
databaseChangeLog:
- changeSet:
id: add-ocr-by-default-to-dossier-template
author: andreiisvoran
changes:
- addColumn:
columns:
- column:
name: ocr_by_default
type: BOOLEAN
defaultValue: false
tableName: dossier_template

14
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java
View File

@ -44,18 +44,29 @@ public class DossierTemplateTesterAndProvider {
public DossierTemplateModel provideTestTemplate() {
return provideTestTemplate("Template 1");
return provideTestTemplate("Template 1", false);
}
public DossierTemplateModel provideTestTemplate(boolean ocrByDefault) {
return provideTestTemplate("Template 1", ocrByDefault);
}
public DossierTemplateModel provideTestTemplate(String name) {
return provideTestTemplate(name, false);
}
public DossierTemplateModel provideTestTemplate(String name, boolean ocrByDefault) {
var cru = new DossierTemplateModel();
cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL));
cru.setName(name);
cru.setDescription("Template 1");
cru.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
cru.setValidTo(OffsetDateTime.now().plusHours(1).truncatedTo(ChronoUnit.MILLIS));
cru.setOcrByDefault(ocrByDefault);
var result = dossierTemplateClient.createOrUpdateDossierTemplate(cru);
@ -70,7 +81,6 @@ public class DossierTemplateTesterAndProvider {
loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId());
return loadedTemplate;
}

32
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java
View File

@ -3,19 +3,20 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.InputStream;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
@ -41,6 +42,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkMo
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
@ -719,4 +721,30 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
// }
}
@Test
public void testDossierTemplateWithOcrByDefault() {
String name = "Test Dossier Template";
DossierTemplateModel dossierTemplateModel = new DossierTemplateModel();
dossierTemplateModel.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL));
dossierTemplateModel.setName(name);
dossierTemplateModel.setDescription("Template 1");
dossierTemplateModel.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
dossierTemplateModel.setValidTo(OffsetDateTime.now().plusHours(1).truncatedTo(ChronoUnit.MILLIS));
dossierTemplateModel.setOcrByDefault(true);
DossierTemplateModel result = dossierTemplateClient.createOrUpdateDossierTemplate(dossierTemplateModel);
assertThat(result.getName()).isEqualTo(name);
assertThat(result.isOcrByDefault()).isTrue();
DossierTemplateModel loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getDossierTemplateId());
assertThat(loadedTemplate).isEqualTo(result);
dossierTemplateModel.setName("Test Dossier Template Update");
dossierTemplateModel.setOcrByDefault(false);
loadedTemplate = dossierTemplateClient.createOrUpdateDossierTemplate(dossierTemplateModel);
assertThat(loadedTemplate.isOcrByDefault()).isFalse();
}
}

40
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -2,6 +2,8 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
@ -14,7 +16,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
@ -30,13 +35,20 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequest;
@ -46,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Lega
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequest;
import feign.FeignException;
import lombok.SneakyThrows;
public class FileTest extends AbstractPersistenceServerServiceTest {
@ -94,6 +107,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
@Autowired
private UserProvider userProvider;
@Autowired
private DossierTemplateClient dossierTemplateClient;
@Autowired
private FileManagementStorageService fileManagementStorageService;
@Test
public void testFileSoftDeleteReupload() {
@ -561,4 +580,25 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getLastReviewer()).isNull();
assertThat(loadedFile.getLastApprover()).isNull();
}
@Test
@SneakyThrows
public void testUploadFileIsInOcrProcessingWithOcrByDefaultFlagTrue() {
DossierTemplateModel dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(true);
Dossier dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "1");
String fileName = "test";
HashFunction hashFunction = Hashing.murmur3_128();
String fileId = hashFunction.hashBytes(((fileName + ".pdf") + dossier.getId()).getBytes(StandardCharsets.UTF_8)).toString();
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.ORIGIN, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
MockMultipartFile mff = new MockMultipartFile(fileName, fileName + ".pdf", "application/pdf", "lorem ipsum".getBytes());
FileUploadResult uploadResult = uploadClient.upload(mff, dossier.getId(), false);
fileId = uploadResult.getFileIds().iterator().next();
FileStatus file = fileClient.getFileStatus(dossier.getId(), fileId);
assertThat(file.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED);
}
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierTemplateModel.java
View File

@ -65,6 +65,8 @@ public class DossierTemplateModel {
@Schema(description = "Representing the setting if dictionary updates is applied to all dossiers")
private boolean applyDictionaryUpdatesToAllDossiersByDefault;
@Schema(description = "Flag that specifies if OCR is automatically performed on upload for all dossiers of this template")
private boolean ocrByDefault;
public String getId() {

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/CreateOrUpdateDossierTemplateRequest.java
View File

@ -37,4 +37,6 @@ public class CreateOrUpdateDossierTemplateRequest {
private boolean applyDictionaryUpdatesToAllDossiersByDefault;
private boolean ocrByDefault;
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierTemplate.java
View File

@ -34,4 +34,6 @@ public class DossierTemplate {
private boolean keepHiddenText;
private boolean keepOverlappingObjects;
private boolean applyDictionaryUpdatesToAllDossiersByDefault;
private boolean ocrByDefault;
}