Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #111: Fixed optimize issue

Browse Source 
Merge in RED/persistence-service from pdf-optimize-on-upload-error to master

* commit 'ec292dacbeccb18fe5266cff8929b4dab20cee45':
  conditional reprocess
  handle optimize and infinite loop on analysis
  Fixed optimize issue
This commit is contained in:
Timo Bejan 2021-11-10 15:25:47 +01:00
commit 91828d6f98
7 changed files with 61 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -38,7 +38,8 @@ public class FileModel {
private boolean hasSuggestions;
private boolean hasImages;
private boolean hasUpdates;
private boolean analysisRequired;
private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
private String uploader;
private long dictionaryVersion;
private long rulesVersion;
@ -59,4 +60,8 @@ public class FileModel {
private String dossierTemplateId;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -77,7 +77,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastManualRedaction = :lastManualRedaction, " +
"f.hasSuggestions = :hasSuggestions where f.id = :fileId")
void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastManualRedaction,
boolean hasSuggestions);
boolean hasSuggestions);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastManualRedaction = :lastManualRedaction, " +
@ -89,7 +89,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " +
"f.deleted = :softDeletedTime where f.id = :fileId")
int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
OffsetDateTime softDeletedTime);
OffsetDateTime softDeletedTime);
@Modifying
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +

4
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java
View File

@ -1,11 +1,9 @@
package com.iqser.red.service.peristence.v1.server.service;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java
View File

@ -10,12 +10,11 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -74,8 +73,9 @@ public class FileService {
}
// if successful update request with linearized PDF
request.setData(optimized.getDocument());
} catch (Exception e) {
} catch (FeignException e) {
log.warn("Failed to optimize file: {}", request.getFilename(), e);
throw new BadRequestException("Failed to optimize document");
}
fileManagementStorageService.storeObject(request.getDossierId(), fileId, FileType.ORIGIN, request.getData());

54
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java
View File

@ -7,7 +7,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -39,46 +40,50 @@ public class ReanalysisRequiredStatusService {
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
Map<String, Long> dossierVersionMap = new HashMap<>();
Map<String, DossierEntity> dossierMap = new HashMap<>();
fileModels.forEach(entry -> entry.setAnalysisRequired(computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap)));
fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
});
return fileModels;
}
private boolean computeAnalysisRequired(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
if(ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())){
return true;
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
return new AnalysisRequiredResult(true, true);
}
if(ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
switch (fileStatus.getWorkflowStatus()) {
case UNASSIGNED:
case UNDER_REVIEW:
case UNDER_APPROVAL:
if (fileStatus.getLastProcessed() == null) {
return true;
return new AnalysisRequiredResult(true, true);
}
if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
return true;
return new AnalysisRequiredResult(true, true);
} else {
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
}
default:
return false;
return new AnalysisRequiredResult(false, false);
}
}
return false;
return new AnalysisRequiredResult(false, false);
}
private boolean requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
// enhance with dossierTemplateId
DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId()));
@ -96,16 +101,16 @@ public class ReanalysisRequiredStatusService {
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
var dossierDictionaryVersionMatches = fileStatus.getDossierDictionaryVersion() == dossierDictionaryVersion;
var analysisRequired = !(rulesVersionMatches && dictionaryVersionMatches && legalBasisVersionMatches && dossierDictionaryVersionMatches);
if (analysisRequired) {
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches;
var fullAnalysisRequired = !rulesVersionMatches || !legalBasisVersionMatches;
if (reanalysisRequired || fullAnalysisRequired) {
log.info("For file: {} analysis is required because -> ruleVersionMatches: {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L),
fileStatus.getDictionaryVersion(), dossierTemplateVersions.getOrDefault(DICTIONARY, -1L),
fileStatus.getLegalBasisVersion(), dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L),
fileStatus.getDossierDictionaryVersion(), dossierDictionaryVersion);
}
return analysisRequired;
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
}
@ -129,4 +134,11 @@ public class ReanalysisRequiredStatusService {
RULES, DICTIONARY, LEGAL_BASIS
}
@Data
@AllArgsConstructor
public static class AnalysisRequiredResult {
private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
}
}

21
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java
View File

@ -8,17 +8,14 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.stream.Collectors;
@ -65,7 +62,13 @@ public class AutomaticAnalysisScheduler {
// in case the file doesn't have numberOfPages set, we assume an average.
worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
log.info("Queued file: {} for automatic analysis!", next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
if (next.isReanalysisRequired()) {
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
}
if (next.isAnalysisRequired()) {
fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), 1);
}
}
}
@ -78,11 +81,11 @@ public class AutomaticAnalysisScheduler {
private List<FileModel> getAllStatuses() {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses()
.stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.filter(f -> !isProcessing(f))
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()))
.stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.filter(f -> !isProcessing(f))
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()))
.stream()
.filter(FileModel::isAnalysisRequired)
.collect(Collectors.toList());

13
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
@ -12,20 +10,17 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import feign.FeignException;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
public class FileTest extends AbstractPersistenceServerServiceTest {
@ -57,8 +52,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
try {
uploadClient.upload(upload);
} catch (BadRequestException e) {
assertThat(e.getMessage().startsWith("empty")).isTrue();
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
}
@ -72,7 +67,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(),"1");
fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(), "1");
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1");
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);