diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 540bbeda2..aca905805 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -88,6 +88,12 @@ + + com.knecon.fforesight + layoutparser-service-internal-api + 0.3.0 + + com.iqser.red.service search-service-api-v1 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java index 601d1b5d1..2d6bc486e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java @@ -1,5 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.configuration; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; + import org.springframework.amqp.core.Queue; import org.springframework.amqp.core.QueueBuilder; import org.springframework.context.annotation.Bean; @@ -58,6 +62,8 @@ public class MessagingConfiguration { public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue"; public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql"; + public static final String X_ERROR_INFO_HEADER = "x-error-message"; + public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp"; @Bean public Queue nerRequestQueue() { @@ -311,4 +317,25 @@ public class MessagingConfiguration { .build(); } + @Bean + public Queue layoutparsingRequestQueue() { + + return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingResponseQueue() { + + return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingDLQ() { + + return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build(); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 8ff33992d..16819aa9d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -40,12 +40,6 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { - case STRUCTURE_ANALYSE: - - //TODO This might be also priority depending on what was the pervious call. - fileStatusService.setStatusAnalyse(dossierId, fileId, false); - break; - case SURROUNDING_TEXT: fileStatusService.setStatusProcessed(analyzeResult.getFileId()); manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions()); @@ -75,7 +69,6 @@ public class FileStatusProcessingUpdateService { } @Transactional - public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse) { fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(), untouchedDocumentResponse.getFileSize()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index b6a93ffc7..cce7a580f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; + import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -12,7 +14,6 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; @@ -22,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysi import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest; +import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; @@ -74,6 +76,7 @@ public class FileStatusService { private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final ViewedPagesPersistenceService viewedPagesPersistenceService; private final FileManagementServiceSettings fileManagementServiceSettings; + private final LayoutParsingRequestFactory layoutParsingRequestFactory; @Transactional @@ -173,7 +176,7 @@ public class FileStatusService { } var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); if (settings.isOcrByDefault() && fileModel.getOcrEndTime() == null) { log.debug("Add file: {} from dossier {} to OCR queue", fileId, dossierId); @@ -181,22 +184,20 @@ public class FileStatusService { return; } - MessageType messageType = null; - - if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { - messageType = MessageType.STRUCTURE_ANALYSE; + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) { + var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority, dossier); + rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); + return; } - if (messageType == null && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; } - if (messageType == null) { - boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; - messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); - } + boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; + MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 98ac5cd1d..26d1d0597 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -10,7 +10,6 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; @@ -39,7 +38,6 @@ public class ReanalysisService { private final IndexingService indexingService; private final PDFTronClient pDFTronRedactionClient; private final FileManagementStorageService fileManagementStorageService; - private final ObjectMapper objectMapper; public void reanalyzeDossier(String dossierId, boolean force) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java new file mode 100644 index 000000000..bf2cc68eb --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing; + +import java.util.Map; +import java.util.Optional; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class LayoutParsingRequestFactory { + + private final FileManagementStorageService fileManagementStorageService; + private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; + + + public LayoutParsingRequest build(String dossierId, String fileId, boolean priority, DossierEntity dossier) { + + Optional optionalImageFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)) : Optional.empty(); + + Optional optionalTableFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.TABLES) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TABLES)) : Optional.empty(); + + return LayoutParsingRequest.builder() + .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority, dossier.getDossierTemplateId())) + .originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) + .imagesFileStorageId(optionalImageFileId) + .tablesFileStorageId(optionalTableFileId) + .pageFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) + .structureFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) + .textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) + .positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)) + .build(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java new file mode 100644 index 000000000..4e678e591 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing; + +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; + +@Service +public class LayoutParsingRequestIdentifierService { + + + private enum IdentifierNames { + DOSSIER_ID, + FILE_ID, + PRIORITY, + DOSSIER_TEMPLATE_ID + } + + + public String parseDossierTemplateId(Map identifiers) { + + return identifiers.get(IdentifierNames.DOSSIER_TEMPLATE_ID.name()); + } + + + public String parseDossierId(Map identifiers) { + + return identifiers.get(IdentifierNames.DOSSIER_ID.name()); + } + + + public String parseFileId(Map identifiers) { + + return identifiers.get(IdentifierNames.FILE_ID.name()); + } + + + public Boolean parsePriority(Map identifiers) { + + return Boolean.parseBoolean(identifiers.get(IdentifierNames.PRIORITY.name())); + } + + + public Map buildIdentifier(String dossierId, String fileId, boolean priority, String dossierTemplateId) { + + return Map.of(IdentifierNames.DOSSIER_TEMPLATE_ID.name(), + dossierTemplateId, + IdentifierNames.DOSSIER_ID.name(), + dossierId, + IdentifierNames.FILE_ID.name(), + fileId, + IdentifierNames.PRIORITY.name(), + String.valueOf(priority)); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java similarity index 90% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java index 0159e5092..2cd6f3f05 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -10,6 +10,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java similarity index 92% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index a4b573931..00cd9ea22 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.io.IOException; import java.time.OffsetDateTime; @@ -13,6 +13,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java new file mode 100644 index 000000000..4313d11a8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -0,0 +1,63 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.queue; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class LayoutParsingFinishedMessageReceiver { + + private final FileStatusService fileStatusService; + private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObjectMapper objectMapper; + private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; + + + @SneakyThrows + @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE) + public void receive(LayoutParsingFinishedEvent response) { + + fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()), + layoutParsingRequestIdentifierService.parseFileId(response.identifier()), + layoutParsingRequestIdentifierService.parsePriority(response.identifier())); + + log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); + } + + + @SneakyThrows + @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_DLQ) + public void handleDLQMessage(Message failedMessage) { + + var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), AnalyzeRequest.class); + log.info("Failed to process analyze request: {}", analyzeRequest); + String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER); + OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER); + timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp); + fileStatusProcessingUpdateService.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp)); + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java similarity index 91% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java index e332a5df5..06150cde6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.io.IOException; import java.time.OffsetDateTime; @@ -12,6 +12,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java similarity index 93% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java index c9ce0abeb..9fa4a0d6e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitListener; @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java new file mode 100644 index 000000000..a254099ac --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java @@ -0,0 +1,205 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; +import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; +import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; +import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.FileModelMapper; +import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; + +class ReanalysisServiceTest { + + private ReanalysisService reanalysisService; + + private FileStatusService fileStatusService; + + @Mock + private ReanalysisRequiredStatusService reanalysisRequiredStatusService; + + @Mock + private IndexingService indexingService; + @Mock + private DossierPersistenceService dossierPersistenceService; + @Mock + private PDFTronClient pdfTronClient; + @Mock + private RabbitTemplate rabbitTemplate; + @Mock + private FileStatusPersistenceService fileStatusPersistenceService; + @Mock + private ObjectMapper objectMapper; + @Mock + private ManualRedactionProviderService manualRedactionProviderService; + @Mock + private FileManagementStorageService fileManagementStorageService; + @Mock + private LegalBasisChangePersistenceService legalBasisChangePersistenceService; + @Mock + private ImageRecategorizationPersistenceService imageRecategorizationPersistenceService; + @Mock + private CommentPersistenceService commentPersistenceService; + @Mock + private ForceRedactionPersistenceService forceRedactionPersistenceService; + @Mock + private RemoveRedactionPersistenceService removeRedactionPersistenceService; + @Mock + private AddRedactionPersistenceService addRedactionPersistenceService; + @Mock + private ResizeRedactionPersistenceService resizeRedactionPersistenceService; + @Mock + private FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + @Mock + private FileManagementServiceSettings settings; + @Mock + private ViewedPagesPersistenceService viewedPagesPersistenceService; + @Mock + private FileManagementServiceSettings fileManagementServiceSettings; + @Mock + private DictionaryPersistenceService dictionaryPersistenceService; + @Mock + private RulesPersistenceService rulesPersistenceService; + @Mock + private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + private static String DOSSIER_ID = "123"; + private static String FILE_ID = "456"; + + private FileModel fileModel; + + + @BeforeEach + public void stubAndCreate() { + + fileModel = MagicConverter.convert(getTestFileEntity(), FileModel.class, new FileModelMapper()); + MockitoAnnotations.openMocks(this); + + fileStatusService = new FileStatusService(fileStatusPersistenceService, + dossierPersistenceService, + rabbitTemplate, + ObjectMapperFactory.create(), + manualRedactionProviderService, + fileManagementStorageService, + legalBasisChangePersistenceService, + imageRecategorizationPersistenceService, + commentPersistenceService, + forceRedactionPersistenceService, + removeRedactionPersistenceService, + addRedactionPersistenceService, + resizeRedactionPersistenceService, + fileAttributeConfigPersistenceService, + settings, + reanalysisRequiredStatusService, + viewedPagesPersistenceService, + fileManagementServiceSettings); + reanalysisService = new ReanalysisService(fileStatusService, dossierPersistenceService, indexingService, pdfTronClient, fileManagementStorageService); + + when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(getTestFileEntity()); + when(dossierPersistenceService.getAndValidateDossier(DOSSIER_ID)).thenReturn(getDossierEntity()); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.ORIGIN)).thenReturn(true); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.TEXT)).thenReturn(true); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES)).thenReturn(true); + when(settings.isFigureDetectionEnabled()).thenReturn(false); + when(settings.isCvTableParsingEnabled()).thenReturn(false); + when(settings.isImageServiceEnabled()).thenReturn(false); + when(settings.isOcrByDefault()).thenReturn(false); + when(dossierPersistenceService.findByDossierId(DOSSIER_ID)).thenReturn(getDossierEntity()); + when(reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true)) // + .thenReturn(fileModel); + } + + + private Dossier getTestDossier() { + + return Dossier.builder().id(DOSSIER_ID).build(); + } + + + private static FileStatus getTestFileStatus() { + + return FileStatus.builder().fileId(FILE_ID).dossierId(DOSSIER_ID).processingStatus(ProcessingStatus.PROCESSED).build(); + } + + + private static FileEntity getTestFileEntity() { + + return FileEntity.builder() + .id(FILE_ID) + .dossierId(DOSSIER_ID) + .workflowStatus(WorkflowStatus.NEW) + .fileAttributes(Collections.emptyList()) + .excluded(false) + .processingStatus(ProcessingStatus.PROCESSED) + .build(); + } + + + private static DossierEntity getDossierEntity() { + + return DossierEntity.builder().id(DOSSIER_ID).build(); + } + + + @Test + public void reanalysisTriggeredByHighLightConversionTest() { + + var dossier = getTestDossier(); + var fileStatus = getTestFileStatus(); + + var textHighlightRequest = new TextHighlightConversionRequest(dossier.getDossierId(), + fileStatus.getFileId(), + Set.of("1", "2", "3"), + TextHighlightConversionOperation.CONVERT); + when(pdfTronClient.convertTextHighlights(textHighlightRequest)).thenReturn(true); + + reanalysisService.convertTextHighlights(textHighlightRequest); + assertThat(fileModel.getAnalysisVersion()).isEqualTo(1); + verify(rabbitTemplate, times(1)).convertAndSend((String) any(), (Object) any()); + verify(fileStatusPersistenceService, times(1)).updateProcessingStatus(FILE_ID, ProcessingStatus.FULL_PROCESSING); + } + +} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 894f85630..6eb0b98fc 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -135,20 +135,23 @@ public class FileStatus { private boolean hasHighlights; @Schema(description = "Size of the optimized, internally stored file.") private Long fileSize; - @Schema(description = "Analysis Version.") + @Schema(description = "Analysis Version.") private int analysisVersion; @Schema(description = "Last time the file was indexed in ES.") private OffsetDateTime lastIndexed; @Schema(description = "The error information for the error state of the file") private FileErrorInfo fileErrorInfo; + @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { return ocrEndTime != null ? ocrEndTime : ocrStartTime; } - public String getId(){ + + public String getId() { + return fileId; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index a234006d6..5bc11524b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -9,14 +9,18 @@ public enum FileType { REDACTION_LOG(".json"), SIMPLIFIED_TEXT(".json"), SECTION_GRID(".json"), - TEXT(".json"), + TEXT(".json"), // TODO: refactor this away NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json"), FIGURE(".json"), TABLES(".json"), - COMPONENTS(".json"); + COMPONENTS(".json"), + DOCUMENT_TEXT(".json"), + DOCUMENT_STRUCTURE(".json"), + DOCUMENT_POSITION(".json"), + DOCUMENT_PAGES(".json"); @Getter private final String extension;