From 342b2dddcd7699aa6e23e7c8dbd8f2d5c8a0e7be Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 19:51:35 +0300 Subject: [PATCH] Hotfix RED-3800 reanslysis required status updates --- .../controller/ReanalysisController.java | 4 +- .../v1/server/service/FileStatusService.java | 12 +-- .../service/ManualRedactionService.java | 27 +++--- .../ReanalysisRequiredStatusService.java | 19 +++- .../tests/ManualRedactionTest.java | 94 ++++++++++++++----- 5 files changed, 109 insertions(+), 47 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index 6267635f6..5ef803f0d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -230,11 +230,11 @@ public class ReanalysisController implements ReanalysisResource { if (force) { filesToReanalyse.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } else { filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index e2330f0c8..f081914cf 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -184,8 +184,6 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); - if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); @@ -201,14 +199,16 @@ public class FileStatusService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); - var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel); - if (fileModel.isExcluded()) { - log.debug("File {} is excluded", fileModel.getId()); + if (fileEntity.isExcluded()) { + log.debug("File {} is excluded", fileEntity.getId()); return; } + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + + boolean reanalyse = fileModel.isReanalysisRequired(); if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index fd1aaa0e2..ff9716244 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -127,23 +127,26 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); + }else { + + var manualTextRedactions = convert(response.stream() + .map(r -> getAddRedaction(fileId, r.getAnnotationId())) + .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); + + if (!manualTextRedactions.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); + } + } - var manualTextRedactions = convert(response.stream() - .map(r -> getAddRedaction(fileId, r.getAnnotationId())) - .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) - .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); - - if (!manualTextRedactions.isEmpty()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - return response; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index c86b3bcca..00f81533a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService { public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { - return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next(); + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next(); + } + + public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) { + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next(); } public List enhanceFileStatusWithAnalysisRequirements(List fileModels) { + return enhanceFileStatusWithAnalysisRequirements(fileModels, false); + } + + public List enhanceFileStatusWithAnalysisRequirements(List fileModels, boolean ignoreProcessingStates) { Map> dossierTemplateVersionMap = new HashMap<>(); Map dossierVersionMap = new HashMap<>(); Map dossierMap = new HashMap<>(); fileModels.forEach(entry -> { - var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap); + var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap); entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired()); entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); }); @@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService { } private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus, + boolean ignoreProcessingStates, Map> dossierTemplateVersionMap, Map dossierVersionMap, Map dossierMap) { - if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) { return new AnalysisRequiredResult(false, true); } - if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) + || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus()) + || ignoreProcessingStates) { switch (fileStatus.getWorkflowStatus()) { case NEW: diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 5c32f5f11..f2fb154a4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1,38 +1,40 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.redaction.v1.model.ManualChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import feign.FeignException; -import org.assertj.core.util.Lists; -import org.junit.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.redaction.v1.model.ManualChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - +import feign.FeignException; import lombok.SneakyThrows; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Autowired private FileManagementStorageService fileManagementStorageService; + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + @Autowired private ObjectMapper objectMapper; @@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } + + @Test + @SneakyThrows + public void testAddToDictionaryRequiresReanalysis() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII"); + + + + // assume file is already proccessed once, test that add to dict triggers reanalysis + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8)); + fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED); + + var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder() + .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) + .section("section test") + .addToDictionary(true) + .addToDossierDictionary(false) + .status(AnnotationStatus.APPROVED) + .typeId(type.getId()) + .user("user") + .reason("1") + .value("test") + .legalBasis("1") + .rectangle(true) + .textAfter("Text After") + .textBefore("Text Before") + .sourceId("SourceId") + .build())).iterator().next(); + + + var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + } + @Test @SneakyThrows public void testManualRedaction() { @@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .annotationStatus(AnnotationStatus.REQUESTED).build()); var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId()); assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED); - assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty()); + assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty()); assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true); manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(), @@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertThat(manualRedactions.getComments()).isNotEmpty(); assertThat(manualRedactions.getResizeRedactions()).isNotEmpty(); - List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); + List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds); // manualRedactions.getForceRedactions() // .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId())));