From 0045f46c7f6a86e1c8d393a178cda06080c5c2e9 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 10 Nov 2021 16:20:06 +0200 Subject: [PATCH] handle optimize and infinite loop on analysis --- .../dossier/file/FileModel.java | 7 ++- .../ReanalysisRequiredStatusService.java | 54 +++++++++++-------- .../scheduler/AutomaticAnalysisScheduler.java | 2 + .../v1/server/integration/tests/FileTest.java | 13 ++--- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index 83bc8aa8b..3642c1f62 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -38,7 +38,8 @@ public class FileModel { private boolean hasSuggestions; private boolean hasImages; private boolean hasUpdates; - private boolean analysisRequired; + private boolean reanalysisRequired; + private boolean fullAnalysisRequired; private String uploader; private long dictionaryVersion; private long rulesVersion; @@ -59,4 +60,8 @@ public class FileModel { private String dossierTemplateId; + public boolean isAnalysisRequired(){ + return this.fullAnalysisRequired || this.reanalysisRequired; + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index a622e2a35..94d20daa3 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -7,7 +7,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; - +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -39,46 +40,50 @@ public class ReanalysisRequiredStatusService { Map> dossierTemplateVersionMap = new HashMap<>(); Map dossierVersionMap = new HashMap<>(); Map dossierMap = new HashMap<>(); - fileModels.forEach(entry -> entry.setAnalysisRequired(computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap))); + fileModels.forEach(entry -> { + var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap); + entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired()); + entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); + }); return fileModels; } - private boolean computeAnalysisRequired(FileModel fileStatus, - Map> dossierTemplateVersionMap, - Map dossierVersionMap, - Map dossierMap) { + private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus, + Map> dossierTemplateVersionMap, + Map dossierVersionMap, + Map dossierMap) { - if(ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())){ - return true; + if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) { + return new AnalysisRequiredResult(true, true); } - if(ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { switch (fileStatus.getWorkflowStatus()) { case UNASSIGNED: case UNDER_REVIEW: case UNDER_APPROVAL: if (fileStatus.getLastProcessed() == null) { - return true; + return new AnalysisRequiredResult(true, true); } if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) { - return true; + return new AnalysisRequiredResult(true, true); } else { return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap); } default: - return false; + return new AnalysisRequiredResult(false, false); } } - return false; + return new AnalysisRequiredResult(false, false); } - private boolean requiresReanalysisBasedOnVersionDifference(FileModel fileStatus, - Map> dossierTemplateVersionMap, - Map dossierVersionMap, - Map dossierMap) { + private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus, + Map> dossierTemplateVersionMap, + Map dossierVersionMap, + Map dossierMap) { // enhance with dossierTemplateId DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId())); @@ -96,16 +101,16 @@ public class ReanalysisRequiredStatusService { var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L); var dossierDictionaryVersionMatches = fileStatus.getDossierDictionaryVersion() == dossierDictionaryVersion; - var analysisRequired = !(rulesVersionMatches && dictionaryVersionMatches && legalBasisVersionMatches && dossierDictionaryVersionMatches); - - if (analysisRequired) { + var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches; + var fullAnalysisRequired = !rulesVersionMatches || !legalBasisVersionMatches; + if (reanalysisRequired || fullAnalysisRequired) { log.info("For file: {} analysis is required because -> ruleVersionMatches: {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L), fileStatus.getDictionaryVersion(), dossierTemplateVersions.getOrDefault(DICTIONARY, -1L), fileStatus.getLegalBasisVersion(), dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L), fileStatus.getDossierDictionaryVersion(), dossierDictionaryVersion); } - return analysisRequired; + return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired); } @@ -129,4 +134,11 @@ public class ReanalysisRequiredStatusService { RULES, DICTIONARY, LEGAL_BASIS } + @Data + @AllArgsConstructor + public static class AnalysisRequiredResult { + private boolean reanalysisRequired; + private boolean fullAnalysisRequired; + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java index eb3bcb2a4..b3e44ec04 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java @@ -66,6 +66,8 @@ public class AutomaticAnalysisScheduler { worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages(); log.info("Queued file: {} for automatic analysis!", next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1); + + fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index c5c1e91cd..ae9dc88cb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient; @@ -12,20 +10,17 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; - +import feign.FeignException; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import java.nio.charset.StandardCharsets; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; public class FileTest extends AbstractPersistenceServerServiceTest { @@ -57,8 +52,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest { try { uploadClient.upload(upload); - } catch (BadRequestException e) { - assertThat(e.getMessage().startsWith("empty")).isTrue(); + } catch (FeignException e) { + assertThat(e.status()).isEqualTo(400); } } @@ -72,7 +67,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(),"1"); + fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1"); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);