Hotfix RED-3800 reanslysis required status updates
This commit is contained in:
Timo Bejan
parent
6d5927f667
commit
342b2dddcd
@ -230,11 +230,11 @@ public class ReanalysisController implements ReanalysisResource {
|
||||
|
||||
if (force) {
|
||||
filesToReanalyse.forEach(file -> {
|
||||
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
|
||||
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
|
||||
});
|
||||
} else {
|
||||
filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> {
|
||||
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
|
||||
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -184,8 +184,6 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
|
||||
|
||||
if (requiresStructureAnalysis) {
|
||||
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
|
||||
@ -201,14 +199,16 @@ public class FileStatusService {
|
||||
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
|
||||
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel);
|
||||
|
||||
if (fileModel.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileModel.getId());
|
||||
if (fileEntity.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileEntity.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
|
||||
|
||||
|
||||
boolean reanalyse = fileModel.isReanalysisRequired();
|
||||
|
||||
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
|
||||
|
||||
@ -127,23 +127,26 @@ public class ManualRedactionService {
|
||||
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
|
||||
}
|
||||
|
||||
|
||||
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
|
||||
|
||||
if (actionPerformed) {
|
||||
// in case of add to dict, there is no need to process surrounding text
|
||||
reprocess(dossierId, fileId);
|
||||
}else {
|
||||
|
||||
var manualTextRedactions = convert(response.stream()
|
||||
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
|
||||
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
|
||||
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
|
||||
|
||||
if (!manualTextRedactions.isEmpty()) {
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
|
||||
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var manualTextRedactions = convert(response.stream()
|
||||
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
|
||||
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
|
||||
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
|
||||
|
||||
if (!manualTextRedactions.isEmpty()) {
|
||||
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
|
||||
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
|
||||
}
|
||||
|
||||
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService {
|
||||
|
||||
|
||||
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
|
||||
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next();
|
||||
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next();
|
||||
}
|
||||
|
||||
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) {
|
||||
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next();
|
||||
}
|
||||
|
||||
public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels) {
|
||||
return enhanceFileStatusWithAnalysisRequirements(fileModels, false);
|
||||
}
|
||||
|
||||
public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels, boolean ignoreProcessingStates) {
|
||||
|
||||
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
|
||||
Map<String, Long> dossierVersionMap = new HashMap<>();
|
||||
Map<String, DossierEntity> dossierMap = new HashMap<>();
|
||||
fileModels.forEach(entry -> {
|
||||
var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
|
||||
var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
|
||||
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
|
||||
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
|
||||
});
|
||||
@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService {
|
||||
}
|
||||
|
||||
private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
|
||||
boolean ignoreProcessingStates,
|
||||
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
|
||||
Map<String, Long> dossierVersionMap,
|
||||
Map<String, DossierEntity> dossierMap) {
|
||||
|
||||
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
|
||||
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) {
|
||||
return new AnalysisRequiredResult(false, true);
|
||||
}
|
||||
|
||||
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
|
||||
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
|
||||
|| ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())
|
||||
|| ignoreProcessingStates) {
|
||||
|
||||
switch (fileStatus.getWorkflowStatus()) {
|
||||
case NEW:
|
||||
|
||||
@ -1,38 +1,40 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import feign.FeignException;
|
||||
import org.assertj.core.util.Lists;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.SneakyThrows;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testAddToDictionaryRequiresReanalysis() {
|
||||
|
||||
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
|
||||
|
||||
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
|
||||
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
|
||||
var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII");
|
||||
|
||||
|
||||
|
||||
// assume file is already proccessed once, test that add to dict triggers reanalysis
|
||||
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8));
|
||||
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8));
|
||||
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8));
|
||||
fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED);
|
||||
|
||||
var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
|
||||
.positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build()))
|
||||
.section("section test")
|
||||
.addToDictionary(true)
|
||||
.addToDossierDictionary(false)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.typeId(type.getId())
|
||||
.user("user")
|
||||
.reason("1")
|
||||
.value("test")
|
||||
.legalBasis("1")
|
||||
.rectangle(true)
|
||||
.textAfter("Text After")
|
||||
.textBefore("Text Before")
|
||||
.sourceId("SourceId")
|
||||
.build())).iterator().next();
|
||||
|
||||
|
||||
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
|
||||
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testManualRedaction() {
|
||||
@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
.annotationStatus(AnnotationStatus.REQUESTED).build());
|
||||
var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId());
|
||||
assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED);
|
||||
assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty());
|
||||
assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty());
|
||||
assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true);
|
||||
|
||||
manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(),
|
||||
@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
|
||||
assertThat(manualRedactions.getComments()).isNotEmpty();
|
||||
assertThat(manualRedactions.getResizeRedactions()).isNotEmpty();
|
||||
|
||||
List<String> annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
|
||||
List<String> annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
|
||||
manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds);
|
||||
// manualRedactions.getForceRedactions()
|
||||
// .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId())));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user