diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 8e06f7a1e..a5e54903f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -43,7 +43,7 @@ public class FileStatusProcessingUpdateService { fileStatusService.setStatusAnalyse(dossierId, fileId, false); //TODO This might be also priority depending on what was the pervious call. - fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null); + fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null, false); } break; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index d23600b64..2dcb8eb34 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -128,7 +128,7 @@ public class FileStatusService { long fileSize) { fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize); - addToAnalysisQueue(dossierId, fileId, false, Set.of()); + addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) { addToPdf2ImageQueue(dossierId, fileId, Set.of()); @@ -181,7 +181,7 @@ public class FileStatusService { @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, - boolean triggeredManually) { + boolean triggeredManually, boolean manualRedactionReanalyse) { log.info("Reprocessing file: {} from dossier {}", fileId, dossierId); @@ -198,7 +198,7 @@ public class FileStatusService { } fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS); - addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse); + addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse, manualRedactionReanalyse); } @@ -231,7 +231,7 @@ public class FileStatusService { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); } - addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet()); + addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false); } @@ -253,14 +253,19 @@ public class FileStatusService { @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority) { - setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false); + setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, false); } + @Transactional + public void setStatusReprocessForManual(String dossierId, String fileId, boolean priority) { + + setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, true); + } @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority, boolean triggeredManually) { - setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually); + setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually, priority); } @@ -268,7 +273,7 @@ public class FileStatusService { public void createStatus(String dossierId, String fileId, String uploader, String filename) { fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader); - addToAnalysisQueue(dossierId, fileId, false, Set.of()); + addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); } @@ -450,7 +455,7 @@ public class FileStatusService { viewedPagesPersistenceService.deleteForFile(fileId); - addToAnalysisQueue(dossierId, fileId, false, Set.of()); + addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); } @@ -511,7 +516,7 @@ public class FileStatusService { } fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE); - addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet()); + addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false); } @@ -529,7 +534,7 @@ public class FileStatusService { @Transactional protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, - Set sectionsToReanalyse) { + Set sectionsToReanalyse, boolean manualRedactionReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); @@ -547,7 +552,7 @@ public class FileStatusService { var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - boolean reanalyse = fileModel.isReanalysisRequired(); + boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); @@ -630,29 +635,11 @@ public class FileStatusService { return MessageType.ANALYSE; } return MessageType.REANALYSE; - } else if (hasUnprocessedManualRedactions(fileModel)) { - return MessageType.REANALYSE; } return MessageType.ANALYSE; } - private boolean hasUnprocessedManualRedactions(FileModel fileModel) { - - return fileModel.getLastProcessed() != null && (manualRedactionProviderService.getManualRedactions(fileModel.getId()) - .getResizeRedactions() - .stream() - .anyMatch(manual -> manual.getRequestDate() - .isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId()) - .getForceRedactions() - .stream() - .anyMatch(manual -> manual.getRequestDate() - .isAfter(fileModel.getLastProcessed())) || manualRedactionProviderService.getManualRedactions(fileModel.getId()) - .getIdsToRemove() - .stream() - .anyMatch(manual -> manual.getRequestDate().isAfter(fileModel.getLastProcessed()))); - } - private void addToOcrQueue(String dossierId, String fileId, int priority) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 61315e585..4176856a0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -1,5 +1,23 @@ package com.iqser.red.service.peristence.v1.server.service; +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.hash.HashFunction; @@ -7,7 +25,14 @@ import com.google.common.hash.Hashing; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -16,8 +41,23 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.CommentRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -26,21 +66,10 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; + import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import javax.transaction.Transactional; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; @Slf4j @Service @@ -174,7 +203,7 @@ public class ManualRedactionService { commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId(); } - if (!removeRedactionRequest.isRemoveFromDictionary()) { + if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) { Optional redactionLogEntryOptional = redactionLog.getRedactionLogEntry() .stream() .filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId())) @@ -463,8 +492,10 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); } - - reprocess(dossierId, fileId); + + if(resizeRedactionRequests.stream().anyMatch(resize -> AnnotationStatus.APPROVED.equals(resize.getStatus()))) { + reprocess(dossierId, fileId); + } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); @@ -726,7 +757,7 @@ public class ManualRedactionService { private void reprocess(String dossierId, String fileId) { - fileStatusService.setStatusReprocess(dossierId, fileId, true); + fileStatusService.setStatusReprocessForManual(dossierId, fileId, true); }