Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

handle optimize and infinite loop on analysis

Browse Source
This commit is contained in:
Timo Bejan 2021-11-10 16:20:06 +02:00
parent fffad3ab80
commit 0045f46c7f
4 changed files with 45 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -38,7 +38,8 @@ public class FileModel {
private boolean hasSuggestions; private boolean hasSuggestions;
private boolean hasImages; private boolean hasImages;
private boolean hasUpdates; private boolean hasUpdates;
private boolean analysisRequired; private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
private String uploader; private String uploader;
private long dictionaryVersion; private long dictionaryVersion;
private long rulesVersion; private long rulesVersion;
@ -59,4 +60,8 @@ public class FileModel {
private String dossierTemplateId; private String dossierTemplateId;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;
}
} }

54
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java
View File

@ -7,7 +7,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -39,46 +40,50 @@ public class ReanalysisRequiredStatusService {
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>(); Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
Map<String, Long> dossierVersionMap = new HashMap<>(); Map<String, Long> dossierVersionMap = new HashMap<>();
Map<String, DossierEntity> dossierMap = new HashMap<>(); Map<String, DossierEntity> dossierMap = new HashMap<>();
fileModels.forEach(entry -> entry.setAnalysisRequired(computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap))); fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
});
return fileModels; return fileModels;
} }
private boolean computeAnalysisRequired(FileModel fileStatus, private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap, Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) { Map<String, DossierEntity> dossierMap) {
if(ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())){ if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
return true; return new AnalysisRequiredResult(true, true);
} }
if(ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
switch (fileStatus.getWorkflowStatus()) { switch (fileStatus.getWorkflowStatus()) {
case UNASSIGNED: case UNASSIGNED:
case UNDER_REVIEW: case UNDER_REVIEW:
case UNDER_APPROVAL: case UNDER_APPROVAL:
if (fileStatus.getLastProcessed() == null) { if (fileStatus.getLastProcessed() == null) {
return true; return new AnalysisRequiredResult(true, true);
} }
if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) { if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
return true; return new AnalysisRequiredResult(true, true);
} else { } else {
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap); return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
} }
default: default:
return false; return new AnalysisRequiredResult(false, false);
} }
} }
return false; return new AnalysisRequiredResult(false, false);
} }
private boolean requiresReanalysisBasedOnVersionDifference(FileModel fileStatus, private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap, Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) { Map<String, DossierEntity> dossierMap) {
// enhance with dossierTemplateId // enhance with dossierTemplateId
DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId())); DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId()));
@ -96,16 +101,16 @@ public class ReanalysisRequiredStatusService {
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L); var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
var dossierDictionaryVersionMatches = fileStatus.getDossierDictionaryVersion() == dossierDictionaryVersion; var dossierDictionaryVersionMatches = fileStatus.getDossierDictionaryVersion() == dossierDictionaryVersion;
var analysisRequired = !(rulesVersionMatches && dictionaryVersionMatches && legalBasisVersionMatches && dossierDictionaryVersionMatches); var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches;
var fullAnalysisRequired = !rulesVersionMatches || !legalBasisVersionMatches;
if (analysisRequired) { if (reanalysisRequired || fullAnalysisRequired) {
log.info("For file: {} analysis is required because -> ruleVersionMatches: {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", log.info("For file: {} analysis is required because -> ruleVersionMatches: {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L), fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L),
fileStatus.getDictionaryVersion(), dossierTemplateVersions.getOrDefault(DICTIONARY, -1L), fileStatus.getDictionaryVersion(), dossierTemplateVersions.getOrDefault(DICTIONARY, -1L),
fileStatus.getLegalBasisVersion(), dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L), fileStatus.getLegalBasisVersion(), dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L),
fileStatus.getDossierDictionaryVersion(), dossierDictionaryVersion); fileStatus.getDossierDictionaryVersion(), dossierDictionaryVersion);
} }
return analysisRequired; return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
} }
@ -129,4 +134,11 @@ public class ReanalysisRequiredStatusService {
RULES, DICTIONARY, LEGAL_BASIS RULES, DICTIONARY, LEGAL_BASIS
} }
@Data
@AllArgsConstructor
public static class AnalysisRequiredResult {
private boolean reanalysisRequired;
private boolean fullAnalysisRequired;
}
} }

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java
View File

@ -66,6 +66,8 @@ public class AutomaticAnalysisScheduler {
worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages(); worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
log.info("Queued file: {} for automatic analysis!", next.getFilename()); log.info("Queued file: {} for automatic analysis!", next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
} }
} }

13
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient; import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
@ -12,20 +10,17 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import feign.FeignException;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
public class FileTest extends AbstractPersistenceServerServiceTest { public class FileTest extends AbstractPersistenceServerServiceTest {
@ -57,8 +52,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
try { try {
uploadClient.upload(upload); uploadClient.upload(upload);
} catch (BadRequestException e) { } catch (FeignException e) {
assertThat(e.getMessage().startsWith("empty")).isTrue(); assertThat(e.status()).isEqualTo(400);
} }
} }
@ -72,7 +67,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(),"1"); fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(), "1");
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1"); assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1");
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);