Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

handle optimize and infinite loop on analysis

Browse Source
This commit is contained in:
Timo Bejan 2021-11-10 16:20:06 +02:00
parent fffad3ab80
commit 0045f46c7f
4 changed files with 45 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -38,7 +38,8 @@ public class FileModel {
private boolean hasSuggestions;
private boolean hasImages;
private boolean hasUpdates;
private boolean analysisRequired;
private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
private String uploader;
private long dictionaryVersion;
private long rulesVersion;
@ -59,4 +60,8 @@ public class FileModel {
private String dossierTemplateId;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;
}
}

54
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java
View File

@ -7,7 +7,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -39,46 +40,50 @@ public class ReanalysisRequiredStatusService {
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
Map<String, Long> dossierVersionMap = new HashMap<>();
Map<String, DossierEntity> dossierMap = new HashMap<>();
fileModels.forEach(entry -> entry.setAnalysisRequired(computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap)));
fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
});
return fileModels;
}
private boolean computeAnalysisRequired(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
if(ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())){
return true;
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
return new AnalysisRequiredResult(true, true);
}
if(ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
switch (fileStatus.getWorkflowStatus()) {
case UNASSIGNED:
case UNDER_REVIEW:
case UNDER_APPROVAL:
if (fileStatus.getLastProcessed() == null) {
return true;
return new AnalysisRequiredResult(true, true);
}
if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
return true;
return new AnalysisRequiredResult(true, true);
} else {
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
}
default:
return false;
return new AnalysisRequiredResult(false, false);
}
}
return false;
return new AnalysisRequiredResult(false, false);
}
private boolean requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
// enhance with dossierTemplateId
DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId()));
@ -96,16 +101,16 @@ public class ReanalysisRequiredStatusService {
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
var dossierDictionaryVersionMatches = fileStatus.getDossierDictionaryVersion() == dossierDictionaryVersion;
var analysisRequired = !(rulesVersionMatches && dictionaryVersionMatches && legalBasisVersionMatches && dossierDictionaryVersionMatches);
if (analysisRequired) {
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches;
var fullAnalysisRequired = !rulesVersionMatches || !legalBasisVersionMatches;
if (reanalysisRequired || fullAnalysisRequired) {
log.info("For file: {} analysis is required because -> ruleVersionMatches: {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L),
fileStatus.getDictionaryVersion(), dossierTemplateVersions.getOrDefault(DICTIONARY, -1L),
fileStatus.getLegalBasisVersion(), dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L),
fileStatus.getDossierDictionaryVersion(), dossierDictionaryVersion);
}
return analysisRequired;
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
}
@ -129,4 +134,11 @@ public class ReanalysisRequiredStatusService {
RULES, DICTIONARY, LEGAL_BASIS
}
@Data
@AllArgsConstructor
public static class AnalysisRequiredResult {
private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
}
}

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java
View File

@ -66,6 +66,8 @@ public class AutomaticAnalysisScheduler {
worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
log.info("Queued file: {} for automatic analysis!", next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
}
}

13
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
@ -12,20 +10,17 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import feign.FeignException;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
public class FileTest extends AbstractPersistenceServerServiceTest {
@ -57,8 +52,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
try {
uploadClient.upload(upload);
} catch (BadRequestException e) {
assertThat(e.getMessage().startsWith("empty")).isTrue();
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
}
@ -72,7 +67,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(),"1");
fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(), "1");
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1");
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);