From 12703c6eda1cac3811fd1dd18ca49e06fb263a6d Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 24 Jul 2023 16:50:20 +0200 Subject: [PATCH 01/11] RED-6725: integrate layoutparsing service --- .../persistence-service-processor-v1/pom.xml | 6 + .../configuration/MessagingConfiguration.java | 27 +++ .../FileStatusProcessingUpdateService.java | 7 - .../processor/service/FileStatusService.java | 23 +- .../processor/service/ReanalysisService.java | 2 - .../LayoutParsingRequestFactory.java | 44 ++++ ...LayoutParsingRequestIdentifierService.java | 57 +++++ .../CvAnalysisMessageReceiver.java | 4 +- .../{ => queue}/ImageMessageReceiver.java | 4 +- .../LayoutParsingFinishedMessageReceiver.java | 63 ++++++ .../{ => queue}/NerMessageReceiver.java | 4 +- .../OCRProcessingMessageReceiver.java | 3 +- .../tests/ReanalysisServiceTest.java | 205 ++++++++++++++++++ .../v1/api/shared/model/FileStatus.java | 7 +- .../dossier/file/FileType.java | 8 +- 15 files changed, 436 insertions(+), 28 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/{ => queue}/CvAnalysisMessageReceiver.java (90%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/{ => queue}/ImageMessageReceiver.java (92%) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/{ => queue}/NerMessageReceiver.java (91%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/{ => queue}/OCRProcessingMessageReceiver.java (93%) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 540bbeda2..aca905805 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -88,6 +88,12 @@ + + com.knecon.fforesight + layoutparser-service-internal-api + 0.3.0 + + com.iqser.red.service search-service-api-v1 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java index 601d1b5d1..2d6bc486e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java @@ -1,5 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.configuration; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; + import org.springframework.amqp.core.Queue; import org.springframework.amqp.core.QueueBuilder; import org.springframework.context.annotation.Bean; @@ -58,6 +62,8 @@ public class MessagingConfiguration { public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue"; public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql"; + public static final String X_ERROR_INFO_HEADER = "x-error-message"; + public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp"; @Bean public Queue nerRequestQueue() { @@ -311,4 +317,25 @@ public class MessagingConfiguration { .build(); } + @Bean + public Queue layoutparsingRequestQueue() { + + return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingResponseQueue() { + + return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingDLQ() { + + return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build(); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 8ff33992d..16819aa9d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -40,12 +40,6 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { - case STRUCTURE_ANALYSE: - - //TODO This might be also priority depending on what was the pervious call. - fileStatusService.setStatusAnalyse(dossierId, fileId, false); - break; - case SURROUNDING_TEXT: fileStatusService.setStatusProcessed(analyzeResult.getFileId()); manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions()); @@ -75,7 +69,6 @@ public class FileStatusProcessingUpdateService { } @Transactional - public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse) { fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(), untouchedDocumentResponse.getFileSize()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 74d7ed993..706e98d72 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; + import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -12,7 +14,6 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; @@ -22,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysi import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest; +import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; @@ -74,6 +76,7 @@ public class FileStatusService { private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final ViewedPagesPersistenceService viewedPagesPersistenceService; private final FileManagementServiceSettings fileManagementServiceSettings; + private final LayoutParsingRequestFactory layoutParsingRequestFactory; @Transactional @@ -174,7 +177,7 @@ public class FileStatusService { } var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); var dossierTemplate = dossier.getDossierTemplate(); if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null) { @@ -183,22 +186,20 @@ public class FileStatusService { return; } - MessageType messageType = null; - - if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { - messageType = MessageType.STRUCTURE_ANALYSE; + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) { + var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority, dossier); + rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); + return; } - if (messageType == null && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; } - if (messageType == null) { - boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; - messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); - } + boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; + MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 7677dec2a..56a125bfc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -10,7 +10,6 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; @@ -39,7 +38,6 @@ public class ReanalysisService { private final IndexingService indexingService; private final PDFTronClient pDFTronRedactionClient; private final FileManagementStorageService fileManagementStorageService; - private final ObjectMapper objectMapper; public void reanalyzeDossier(String dossierId, boolean force) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java new file mode 100644 index 000000000..bf2cc68eb --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing; + +import java.util.Map; +import java.util.Optional; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class LayoutParsingRequestFactory { + + private final FileManagementStorageService fileManagementStorageService; + private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; + + + public LayoutParsingRequest build(String dossierId, String fileId, boolean priority, DossierEntity dossier) { + + Optional optionalImageFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)) : Optional.empty(); + + Optional optionalTableFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.TABLES) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TABLES)) : Optional.empty(); + + return LayoutParsingRequest.builder() + .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority, dossier.getDossierTemplateId())) + .originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) + .imagesFileStorageId(optionalImageFileId) + .tablesFileStorageId(optionalTableFileId) + .pageFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) + .structureFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) + .textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) + .positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)) + .build(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java new file mode 100644 index 000000000..4e678e591 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing; + +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; + +@Service +public class LayoutParsingRequestIdentifierService { + + + private enum IdentifierNames { + DOSSIER_ID, + FILE_ID, + PRIORITY, + DOSSIER_TEMPLATE_ID + } + + + public String parseDossierTemplateId(Map identifiers) { + + return identifiers.get(IdentifierNames.DOSSIER_TEMPLATE_ID.name()); + } + + + public String parseDossierId(Map identifiers) { + + return identifiers.get(IdentifierNames.DOSSIER_ID.name()); + } + + + public String parseFileId(Map identifiers) { + + return identifiers.get(IdentifierNames.FILE_ID.name()); + } + + + public Boolean parsePriority(Map identifiers) { + + return Boolean.parseBoolean(identifiers.get(IdentifierNames.PRIORITY.name())); + } + + + public Map buildIdentifier(String dossierId, String fileId, boolean priority, String dossierTemplateId) { + + return Map.of(IdentifierNames.DOSSIER_TEMPLATE_ID.name(), + dossierTemplateId, + IdentifierNames.DOSSIER_ID.name(), + dossierId, + IdentifierNames.FILE_ID.name(), + fileId, + IdentifierNames.PRIORITY.name(), + String.valueOf(priority)); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java similarity index 90% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java index 0159e5092..2cd6f3f05 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -10,6 +10,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java similarity index 92% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index a4b573931..00cd9ea22 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.io.IOException; import java.time.OffsetDateTime; @@ -13,6 +13,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java new file mode 100644 index 000000000..4313d11a8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -0,0 +1,63 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.queue; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class LayoutParsingFinishedMessageReceiver { + + private final FileStatusService fileStatusService; + private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObjectMapper objectMapper; + private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; + + + @SneakyThrows + @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE) + public void receive(LayoutParsingFinishedEvent response) { + + fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()), + layoutParsingRequestIdentifierService.parseFileId(response.identifier()), + layoutParsingRequestIdentifierService.parsePriority(response.identifier())); + + log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); + } + + + @SneakyThrows + @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_DLQ) + public void handleDLQMessage(Message failedMessage) { + + var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), AnalyzeRequest.class); + log.info("Failed to process analyze request: {}", analyzeRequest); + String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER); + OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER); + timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp); + fileStatusProcessingUpdateService.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp)); + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java similarity index 91% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java index e332a5df5..06150cde6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import java.io.IOException; import java.time.OffsetDateTime; @@ -12,6 +12,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java similarity index 93% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java index c9ce0abeb..9fa4a0d6e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/OCRProcessingMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.queue; import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitListener; @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java new file mode 100644 index 000000000..a254099ac --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java @@ -0,0 +1,205 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; +import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; +import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; +import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.FileModelMapper; +import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; + +class ReanalysisServiceTest { + + private ReanalysisService reanalysisService; + + private FileStatusService fileStatusService; + + @Mock + private ReanalysisRequiredStatusService reanalysisRequiredStatusService; + + @Mock + private IndexingService indexingService; + @Mock + private DossierPersistenceService dossierPersistenceService; + @Mock + private PDFTronClient pdfTronClient; + @Mock + private RabbitTemplate rabbitTemplate; + @Mock + private FileStatusPersistenceService fileStatusPersistenceService; + @Mock + private ObjectMapper objectMapper; + @Mock + private ManualRedactionProviderService manualRedactionProviderService; + @Mock + private FileManagementStorageService fileManagementStorageService; + @Mock + private LegalBasisChangePersistenceService legalBasisChangePersistenceService; + @Mock + private ImageRecategorizationPersistenceService imageRecategorizationPersistenceService; + @Mock + private CommentPersistenceService commentPersistenceService; + @Mock + private ForceRedactionPersistenceService forceRedactionPersistenceService; + @Mock + private RemoveRedactionPersistenceService removeRedactionPersistenceService; + @Mock + private AddRedactionPersistenceService addRedactionPersistenceService; + @Mock + private ResizeRedactionPersistenceService resizeRedactionPersistenceService; + @Mock + private FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; + @Mock + private FileManagementServiceSettings settings; + @Mock + private ViewedPagesPersistenceService viewedPagesPersistenceService; + @Mock + private FileManagementServiceSettings fileManagementServiceSettings; + @Mock + private DictionaryPersistenceService dictionaryPersistenceService; + @Mock + private RulesPersistenceService rulesPersistenceService; + @Mock + private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + + private static String DOSSIER_ID = "123"; + private static String FILE_ID = "456"; + + private FileModel fileModel; + + + @BeforeEach + public void stubAndCreate() { + + fileModel = MagicConverter.convert(getTestFileEntity(), FileModel.class, new FileModelMapper()); + MockitoAnnotations.openMocks(this); + + fileStatusService = new FileStatusService(fileStatusPersistenceService, + dossierPersistenceService, + rabbitTemplate, + ObjectMapperFactory.create(), + manualRedactionProviderService, + fileManagementStorageService, + legalBasisChangePersistenceService, + imageRecategorizationPersistenceService, + commentPersistenceService, + forceRedactionPersistenceService, + removeRedactionPersistenceService, + addRedactionPersistenceService, + resizeRedactionPersistenceService, + fileAttributeConfigPersistenceService, + settings, + reanalysisRequiredStatusService, + viewedPagesPersistenceService, + fileManagementServiceSettings); + reanalysisService = new ReanalysisService(fileStatusService, dossierPersistenceService, indexingService, pdfTronClient, fileManagementStorageService); + + when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(getTestFileEntity()); + when(dossierPersistenceService.getAndValidateDossier(DOSSIER_ID)).thenReturn(getDossierEntity()); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.ORIGIN)).thenReturn(true); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.TEXT)).thenReturn(true); + when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES)).thenReturn(true); + when(settings.isFigureDetectionEnabled()).thenReturn(false); + when(settings.isCvTableParsingEnabled()).thenReturn(false); + when(settings.isImageServiceEnabled()).thenReturn(false); + when(settings.isOcrByDefault()).thenReturn(false); + when(dossierPersistenceService.findByDossierId(DOSSIER_ID)).thenReturn(getDossierEntity()); + when(reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true)) // + .thenReturn(fileModel); + } + + + private Dossier getTestDossier() { + + return Dossier.builder().id(DOSSIER_ID).build(); + } + + + private static FileStatus getTestFileStatus() { + + return FileStatus.builder().fileId(FILE_ID).dossierId(DOSSIER_ID).processingStatus(ProcessingStatus.PROCESSED).build(); + } + + + private static FileEntity getTestFileEntity() { + + return FileEntity.builder() + .id(FILE_ID) + .dossierId(DOSSIER_ID) + .workflowStatus(WorkflowStatus.NEW) + .fileAttributes(Collections.emptyList()) + .excluded(false) + .processingStatus(ProcessingStatus.PROCESSED) + .build(); + } + + + private static DossierEntity getDossierEntity() { + + return DossierEntity.builder().id(DOSSIER_ID).build(); + } + + + @Test + public void reanalysisTriggeredByHighLightConversionTest() { + + var dossier = getTestDossier(); + var fileStatus = getTestFileStatus(); + + var textHighlightRequest = new TextHighlightConversionRequest(dossier.getDossierId(), + fileStatus.getFileId(), + Set.of("1", "2", "3"), + TextHighlightConversionOperation.CONVERT); + when(pdfTronClient.convertTextHighlights(textHighlightRequest)).thenReturn(true); + + reanalysisService.convertTextHighlights(textHighlightRequest); + assertThat(fileModel.getAnalysisVersion()).isEqualTo(1); + verify(rabbitTemplate, times(1)).convertAndSend((String) any(), (Object) any()); + verify(fileStatusPersistenceService, times(1)).updateProcessingStatus(FILE_ID, ProcessingStatus.FULL_PROCESSING); + } + +} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 894f85630..6eb0b98fc 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -135,20 +135,23 @@ public class FileStatus { private boolean hasHighlights; @Schema(description = "Size of the optimized, internally stored file.") private Long fileSize; - @Schema(description = "Analysis Version.") + @Schema(description = "Analysis Version.") private int analysisVersion; @Schema(description = "Last time the file was indexed in ES.") private OffsetDateTime lastIndexed; @Schema(description = "The error information for the error state of the file") private FileErrorInfo fileErrorInfo; + @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { return ocrEndTime != null ? ocrEndTime : ocrStartTime; } - public String getId(){ + + public String getId() { + return fileId; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index a234006d6..5bc11524b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -9,14 +9,18 @@ public enum FileType { REDACTION_LOG(".json"), SIMPLIFIED_TEXT(".json"), SECTION_GRID(".json"), - TEXT(".json"), + TEXT(".json"), // TODO: refactor this away NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json"), FIGURE(".json"), TABLES(".json"), - COMPONENTS(".json"); + COMPONENTS(".json"), + DOCUMENT_TEXT(".json"), + DOCUMENT_STRUCTURE(".json"), + DOCUMENT_POSITION(".json"), + DOCUMENT_PAGES(".json"); @Getter private final String extension; -- 2.47.2 From 49770900702d67b5fb5d218fbcd590d9fee44cb0 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 24 Jul 2023 16:56:52 +0200 Subject: [PATCH 02/11] RED-6725: integrate layoutparsing service * updated layout parser version * added layout parsing type --- .../persistence-service-processor-v1/pom.xml | 2 +- .../layoutparsing/LayoutParsingRequestFactory.java | 12 ++++++++++++ .../src/test/resources/application.yaml | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index aca905805..aedde5f74 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.3.0 + 0.4.0 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index bf2cc68eb..f715e2f37 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.layout import java.util.Map; import java.util.Optional; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; @@ -10,6 +11,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import lombok.RequiredArgsConstructor; @@ -17,12 +19,21 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class LayoutParsingRequestFactory { + @Value("${application.type}") + private String applicationType; + private final FileManagementStorageService fileManagementStorageService; private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; public LayoutParsingRequest build(String dossierId, String fileId, boolean priority, DossierEntity dossier) { + LayoutParsingType type = switch (applicationType) { + case "DocuMine" -> LayoutParsingType.DOCUMINE; + case "TAAS" -> LayoutParsingType.TAAS; + default -> LayoutParsingType.REDACT_MANAGER; + }; + Optional optionalImageFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO) // ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)) : Optional.empty(); @@ -30,6 +41,7 @@ public class LayoutParsingRequestFactory { ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TABLES)) : Optional.empty(); return LayoutParsingRequest.builder() + .layoutParsingType(type) .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority, dossier.getDossierTemplateId())) .originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) .imagesFileStorageId(optionalImageFileId) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml index 329b7f543..c85f2c996 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml @@ -1,5 +1,8 @@ monitoring.enabled: true +application: + type: "RedactManager" + spring: main: allow-circular-references: true # FIXME -- 2.47.2 From c9254532d767e5e2c3a7fea73e780d35a2ba057d Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 24 Jul 2023 17:01:51 +0200 Subject: [PATCH 03/11] RED-6725: fix test mock --- .../server/integration/tests/ReanalysisServiceTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java index a254099ac..2f98d48be 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java @@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Indexin import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; +import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; @@ -44,13 +45,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; class ReanalysisServiceTest { @@ -105,6 +106,8 @@ class ReanalysisServiceTest { private RulesPersistenceService rulesPersistenceService; @Mock private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + @Mock + private LayoutParsingRequestFactory layoutParsingRequestFactory; private static String DOSSIER_ID = "123"; private static String FILE_ID = "456"; @@ -135,7 +138,8 @@ class ReanalysisServiceTest { settings, reanalysisRequiredStatusService, viewedPagesPersistenceService, - fileManagementServiceSettings); + fileManagementServiceSettings, + layoutParsingRequestFactory); reanalysisService = new ReanalysisService(fileStatusService, dossierPersistenceService, indexingService, pdfTronClient, fileManagementStorageService); when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(getTestFileEntity()); -- 2.47.2 From 410556303601ac43bd72194a4c92a6ba20202be2 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 24 Jul 2023 17:58:31 +0200 Subject: [PATCH 04/11] RED-6725: deprecate FileType.TEXT, fix tests --- .../controller/RedactionLogController.java | 108 +++++++++ .../resource/RedactionLogResource.java | 31 ++- .../internal/AdminInterfaceController.java | 10 +- .../processor/service/FileStatusService.java | 13 +- .../tests/ManualRedactionTest.java | 5 +- .../tests/ReanalysisServiceTest.java | 209 ------------------ .../dossier/file/FileType.java | 3 +- 7 files changed, 162 insertions(+), 217 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java index f7c2fc52a..aba7e6fe7 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java @@ -97,6 +97,114 @@ public class RedactionLogController implements RedactionLogResource { } + @SneakyThrows + @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") + public ResponseEntity getDocumentText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { + + try { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); + + var fileStatus = fileStatusService.getStatus(fileId); + String filename = fileStatus.getFilename(); + if (filename != null) { + var index = filename.lastIndexOf("."); + String prefix = filename.substring(0, index); + filename = prefix + ".json"; + httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); + } + + byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_TEXT); + return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); + + } catch (FeignException e) { + throw processFeignException(e); + } + } + + + @SneakyThrows + @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") + public ResponseEntity getDocumentPositions(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { + + try { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); + + var fileStatus = fileStatusService.getStatus(fileId); + String filename = fileStatus.getFilename(); + if (filename != null) { + var index = filename.lastIndexOf("."); + String prefix = filename.substring(0, index); + filename = prefix + ".json"; + httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); + } + + byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_POSITION); + return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); + + } catch (FeignException e) { + throw processFeignException(e); + } + } + + + @SneakyThrows + @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") + public ResponseEntity getDocumentStructure(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { + + try { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); + + var fileStatus = fileStatusService.getStatus(fileId); + String filename = fileStatus.getFilename(); + if (filename != null) { + var index = filename.lastIndexOf("."); + String prefix = filename.substring(0, index); + filename = prefix + ".json"; + httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); + } + + byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_STRUCTURE); + return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); + + } catch (FeignException e) { + throw processFeignException(e); + } + } + + + @SneakyThrows + @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") + public ResponseEntity getDocumentPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { + + try { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); + + var fileStatus = fileStatusService.getStatus(fileId); + String filename = fileStatus.getFilename(); + if (filename != null) { + var index = filename.lastIndexOf("."); + String prefix = filename.substring(0, index); + filename = prefix + ".json"; + httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); + } + + byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_PAGES); + return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); + + } catch (FeignException e) { + throw processFeignException(e); + } + } + + @SneakyThrows @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") public ResponseEntity getSimplifiedSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java index a495b45a2..1c07b8307 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java @@ -26,6 +26,10 @@ public interface RedactionLogResource { String REDACTION_LOG_PATH = ExternalApi.BASE_PATH + "/redactionLog"; String SECTION_GRID_PATH = ExternalApi.BASE_PATH + "/sectionGrid"; String SECTION_TEXT_PATH = ExternalApi.BASE_PATH + "/sectionText"; + String DOCUMENT_TEXT_PATH = ExternalApi.BASE_PATH + "/documentText"; + String DOCUMENT_POSITIONS_PATH = ExternalApi.BASE_PATH + "/documentPositions"; + String DOCUMENT_PAGES_PATH = ExternalApi.BASE_PATH + "/documentPages"; + String DOCUMENT_STRUCTURE_PATH = ExternalApi.BASE_PATH + "/documentStructure"; String SIMPLIFIED_SECTION_TEXT_PATH = ExternalApi.BASE_PATH + "/simplifiedSectionText"; String FILE_ID = "fileId"; @@ -51,12 +55,37 @@ public interface RedactionLogResource { SectionGrid getSectionGrid(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated @GetMapping(value = SECTION_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) - @Operation(summary = "Gets the section text for a fileId", description = "None") + @Operation(summary = "Gets the text blocks of a document for a fileId", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) ResponseEntity getSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + @GetMapping(value = DOCUMENT_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) + @Operation(summary = "Gets the text blocks of a document for a fileId", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + ResponseEntity getDocumentText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + + + @GetMapping(value = DOCUMENT_POSITIONS_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) + @Operation(summary = "Gets the positions of the text blocks of a document for a fileId", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + ResponseEntity getDocumentPositions(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + + + @GetMapping(value = DOCUMENT_STRUCTURE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) + @Operation(summary = "Gets the document structure for a fileId", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + ResponseEntity getDocumentStructure(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + + + @GetMapping(value = DOCUMENT_PAGES_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) + @Operation(summary = "Gets the page information of a document for a fileId", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + ResponseEntity getDocumentPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); + + @GetMapping(value = SIMPLIFIED_SECTION_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the simplified section text for a fileId", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The simplified section text is not found.")}) diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java index f2d2190ee..d205ed4e6 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java @@ -37,7 +37,10 @@ public class AdminInterfaceController { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.FIGURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TABLES); @@ -136,7 +139,10 @@ public class AdminInterfaceController { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 706e98d72..03da1fbf9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -186,8 +186,9 @@ public class FileStatusService { return; } - if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) { + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT)) { var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority, dossier); + setStatusFullProcessing(fileId); rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); return; } @@ -599,7 +600,10 @@ public class FileStatusService { if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); } @@ -637,7 +641,10 @@ public class FileStatusService { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.FIGURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TABLES); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index e334eb7ef..13bec1ae8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -462,7 +462,10 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII"); // assume file is already proccessed once, test that add to dict triggers reanalysis - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.TEXT, "{}"); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.DOCUMENT_TEXT, "{}"); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.DOCUMENT_PAGES, "{}"); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.DOCUMENT_STRUCTURE, "{}"); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.DOCUMENT_POSITION, "{}"); fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "{}"); fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "{}"); fileStatusPersistenceService.updateProcessingStatus(file.getId(), ProcessingStatus.PROCESSED); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java deleted file mode 100644 index 2f98d48be..000000000 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisServiceTest.java +++ /dev/null @@ -1,209 +0,0 @@ -package com.iqser.red.service.peristence.v1.server.integration.tests; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.Collections; -import java.util.Set; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.amqp.rabbit.core.RabbitTemplate; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; -import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; -import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; -import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; -import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; -import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; -import com.iqser.red.service.persistence.management.v1.processor.utils.FileModelMapper; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; - -class ReanalysisServiceTest { - - private ReanalysisService reanalysisService; - - private FileStatusService fileStatusService; - - @Mock - private ReanalysisRequiredStatusService reanalysisRequiredStatusService; - - @Mock - private IndexingService indexingService; - @Mock - private DossierPersistenceService dossierPersistenceService; - @Mock - private PDFTronClient pdfTronClient; - @Mock - private RabbitTemplate rabbitTemplate; - @Mock - private FileStatusPersistenceService fileStatusPersistenceService; - @Mock - private ObjectMapper objectMapper; - @Mock - private ManualRedactionProviderService manualRedactionProviderService; - @Mock - private FileManagementStorageService fileManagementStorageService; - @Mock - private LegalBasisChangePersistenceService legalBasisChangePersistenceService; - @Mock - private ImageRecategorizationPersistenceService imageRecategorizationPersistenceService; - @Mock - private CommentPersistenceService commentPersistenceService; - @Mock - private ForceRedactionPersistenceService forceRedactionPersistenceService; - @Mock - private RemoveRedactionPersistenceService removeRedactionPersistenceService; - @Mock - private AddRedactionPersistenceService addRedactionPersistenceService; - @Mock - private ResizeRedactionPersistenceService resizeRedactionPersistenceService; - @Mock - private FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; - @Mock - private FileManagementServiceSettings settings; - @Mock - private ViewedPagesPersistenceService viewedPagesPersistenceService; - @Mock - private FileManagementServiceSettings fileManagementServiceSettings; - @Mock - private DictionaryPersistenceService dictionaryPersistenceService; - @Mock - private RulesPersistenceService rulesPersistenceService; - @Mock - private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; - @Mock - private LayoutParsingRequestFactory layoutParsingRequestFactory; - - private static String DOSSIER_ID = "123"; - private static String FILE_ID = "456"; - - private FileModel fileModel; - - - @BeforeEach - public void stubAndCreate() { - - fileModel = MagicConverter.convert(getTestFileEntity(), FileModel.class, new FileModelMapper()); - MockitoAnnotations.openMocks(this); - - fileStatusService = new FileStatusService(fileStatusPersistenceService, - dossierPersistenceService, - rabbitTemplate, - ObjectMapperFactory.create(), - manualRedactionProviderService, - fileManagementStorageService, - legalBasisChangePersistenceService, - imageRecategorizationPersistenceService, - commentPersistenceService, - forceRedactionPersistenceService, - removeRedactionPersistenceService, - addRedactionPersistenceService, - resizeRedactionPersistenceService, - fileAttributeConfigPersistenceService, - settings, - reanalysisRequiredStatusService, - viewedPagesPersistenceService, - fileManagementServiceSettings, - layoutParsingRequestFactory); - reanalysisService = new ReanalysisService(fileStatusService, dossierPersistenceService, indexingService, pdfTronClient, fileManagementStorageService); - - when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(getTestFileEntity()); - when(dossierPersistenceService.getAndValidateDossier(DOSSIER_ID)).thenReturn(getDossierEntity()); - when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.ORIGIN)).thenReturn(true); - when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.TEXT)).thenReturn(true); - when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES)).thenReturn(true); - when(settings.isFigureDetectionEnabled()).thenReturn(false); - when(settings.isCvTableParsingEnabled()).thenReturn(false); - when(settings.isImageServiceEnabled()).thenReturn(false); - when(settings.isOcrByDefault()).thenReturn(false); - when(dossierPersistenceService.findByDossierId(DOSSIER_ID)).thenReturn(getDossierEntity()); - when(reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true)) // - .thenReturn(fileModel); - } - - - private Dossier getTestDossier() { - - return Dossier.builder().id(DOSSIER_ID).build(); - } - - - private static FileStatus getTestFileStatus() { - - return FileStatus.builder().fileId(FILE_ID).dossierId(DOSSIER_ID).processingStatus(ProcessingStatus.PROCESSED).build(); - } - - - private static FileEntity getTestFileEntity() { - - return FileEntity.builder() - .id(FILE_ID) - .dossierId(DOSSIER_ID) - .workflowStatus(WorkflowStatus.NEW) - .fileAttributes(Collections.emptyList()) - .excluded(false) - .processingStatus(ProcessingStatus.PROCESSED) - .build(); - } - - - private static DossierEntity getDossierEntity() { - - return DossierEntity.builder().id(DOSSIER_ID).build(); - } - - - @Test - public void reanalysisTriggeredByHighLightConversionTest() { - - var dossier = getTestDossier(); - var fileStatus = getTestFileStatus(); - - var textHighlightRequest = new TextHighlightConversionRequest(dossier.getDossierId(), - fileStatus.getFileId(), - Set.of("1", "2", "3"), - TextHighlightConversionOperation.CONVERT); - when(pdfTronClient.convertTextHighlights(textHighlightRequest)).thenReturn(true); - - reanalysisService.convertTextHighlights(textHighlightRequest); - assertThat(fileModel.getAnalysisVersion()).isEqualTo(1); - verify(rabbitTemplate, times(1)).convertAndSend((String) any(), (Object) any()); - verify(fileStatusPersistenceService, times(1)).updateProcessingStatus(FILE_ID, ProcessingStatus.FULL_PROCESSING); - } - -} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index 5bc11524b..9602ba95b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -9,7 +9,7 @@ public enum FileType { REDACTION_LOG(".json"), SIMPLIFIED_TEXT(".json"), SECTION_GRID(".json"), - TEXT(".json"), // TODO: refactor this away + TEXT(".json"), // deprecated file type, only present in legacy migrations NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), @@ -17,6 +17,7 @@ public enum FileType { FIGURE(".json"), TABLES(".json"), COMPONENTS(".json"), + // document is split into 4 files, all should be overridden/deleted at the same time DOCUMENT_TEXT(".json"), DOCUMENT_STRUCTURE(".json"), DOCUMENT_POSITION(".json"), -- 2.47.2 From 0e2200f33671f3db66d2fe9ba6f665c463f0e802 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 31 Jul 2023 14:04:43 +0200 Subject: [PATCH 05/11] RED-6725: integrate layout parser * fixed error messages * extracted common zip file response entity function * added content length header * removed unused identifier --- .../controller/RedactionLogController.java | 105 +++++------------- .../resource/RedactionLogResource.java | 18 +-- .../persistence-service-processor-v1/pom.xml | 2 +- .../processor/service/FileStatusService.java | 1 - .../LayoutParsingRequestFactory.java | 2 +- ...LayoutParsingRequestIdentifierService.java | 18 +-- 6 files changed, 38 insertions(+), 108 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java index aba7e6fe7..f5406748f 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java @@ -75,49 +75,39 @@ public class RedactionLogController implements RedactionLogResource { public ResponseEntity getSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { try { - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); - - var fileStatus = fileStatusService.getStatus(fileId); - String filename = fileStatus.getFilename(); - if (filename != null) { - var index = filename.lastIndexOf("."); - String prefix = filename.substring(0, index); - filename = prefix + ".json"; - httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); - } - - byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.TEXT); - return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); - + return buildZipFileResponseEntity(fileId, dossierId, FileType.TEXT); } catch (FeignException e) { throw processFeignException(e); } } + private ResponseEntity buildZipFileResponseEntity(String fileId, String dossierId, FileType fileType) throws IOException { + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); + + var fileStatus = fileStatusService.getStatus(fileId); + String filename = fileStatus.getFilename(); + if (filename != null) { + var index = filename.lastIndexOf("."); + String prefix = filename.substring(0, index); + filename = prefix + ".json"; + httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=utf-8''" + StringEncodingUtils.urlEncode(prefix) + ".zip"); + } + + byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, fileType); + httpHeaders.setContentLength(zipBytes.length); + return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); + } + + @SneakyThrows @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") public ResponseEntity getDocumentText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { try { - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); - - var fileStatus = fileStatusService.getStatus(fileId); - String filename = fileStatus.getFilename(); - if (filename != null) { - var index = filename.lastIndexOf("."); - String prefix = filename.substring(0, index); - filename = prefix + ".json"; - httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); - } - - byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_TEXT); - return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); - + return buildZipFileResponseEntity(fileId, dossierId, FileType.DOCUMENT_TEXT); } catch (FeignException e) { throw processFeignException(e); } @@ -129,22 +119,7 @@ public class RedactionLogController implements RedactionLogResource { public ResponseEntity getDocumentPositions(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { try { - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); - - var fileStatus = fileStatusService.getStatus(fileId); - String filename = fileStatus.getFilename(); - if (filename != null) { - var index = filename.lastIndexOf("."); - String prefix = filename.substring(0, index); - filename = prefix + ".json"; - httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); - } - - byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_POSITION); - return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); - + return buildZipFileResponseEntity(fileId, dossierId, FileType.DOCUMENT_POSITION); } catch (FeignException e) { throw processFeignException(e); } @@ -156,22 +131,7 @@ public class RedactionLogController implements RedactionLogResource { public ResponseEntity getDocumentStructure(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { try { - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); - - var fileStatus = fileStatusService.getStatus(fileId); - String filename = fileStatus.getFilename(); - if (filename != null) { - var index = filename.lastIndexOf("."); - String prefix = filename.substring(0, index); - filename = prefix + ".json"; - httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); - } - - byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_STRUCTURE); - return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); - + return buildZipFileResponseEntity(fileId, dossierId, FileType.DOCUMENT_STRUCTURE); } catch (FeignException e) { throw processFeignException(e); } @@ -183,22 +143,7 @@ public class RedactionLogController implements RedactionLogResource { public ResponseEntity getDocumentPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { try { - - HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.setContentType(MediaType.parseMediaType("application/zip")); - - var fileStatus = fileStatusService.getStatus(fileId); - String filename = fileStatus.getFilename(); - if (filename != null) { - var index = filename.lastIndexOf("."); - String prefix = filename.substring(0, index); - filename = prefix + ".json"; - httpHeaders.add("Content-Disposition", "attachment; filename=" + prefix + ".zip"); - } - - byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, FileType.DOCUMENT_PAGES); - return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK); - + return buildZipFileResponseEntity(fileId, dossierId, FileType.DOCUMENT_PAGES); } catch (FeignException e) { throw processFeignException(e); } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java index 1c07b8307..54dc7f04c 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java @@ -41,7 +41,7 @@ public interface RedactionLogResource { @GetMapping(value = REDACTION_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Gets the redaction log for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")}) RedactionLog getRedactionLog(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(value = "excludedType", required = false) List excludedTypes, @@ -51,50 +51,50 @@ public interface RedactionLogResource { @GetMapping(value = SECTION_GRID_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Gets the section grid for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section grid is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The section grid is not found.")}) SectionGrid getSectionGrid(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @Deprecated @GetMapping(value = SECTION_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the text blocks of a document for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) ResponseEntity getSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @GetMapping(value = DOCUMENT_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the text blocks of a document for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The document text is not found.")}) ResponseEntity getDocumentText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @GetMapping(value = DOCUMENT_POSITIONS_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the positions of the text blocks of a document for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The document positions is not found.")}) ResponseEntity getDocumentPositions(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @GetMapping(value = DOCUMENT_STRUCTURE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the document structure for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The document structure is not found.")}) ResponseEntity getDocumentStructure(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @GetMapping(value = DOCUMENT_PAGES_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the page information of a document for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The page information is not found.")}) ResponseEntity getDocumentPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @GetMapping(value = SIMPLIFIED_SECTION_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE) @Operation(summary = "Gets the simplified section text for a fileId", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The simplified section text is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The simplified section text is not found.")}) ResponseEntity getSimplifiedSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId); @PostMapping(value = REDACTION_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/filtered", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Gets the redaction log for a fileId grater than the specified date", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request " + "contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")}) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")}) RedactionLog getFilteredRedactionLog(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FilteredRedactionLogRequest filteredRedactionLogRequest); diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index aedde5f74..056486ba5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.4.0 + RED-6725-2 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 03da1fbf9..d4c97a921 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -61,7 +61,6 @@ public class FileStatusService { private final FileStatusPersistenceService fileStatusPersistenceService; private final DossierPersistenceService dossierPersistenceService; private final RabbitTemplate rabbitTemplate; - private final ObjectMapper objectMapper; private final ManualRedactionProviderService manualRedactionProviderService; private final FileManagementStorageService fileManagementStorageService; private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index f715e2f37..5c268160d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -42,7 +42,7 @@ public class LayoutParsingRequestFactory { return LayoutParsingRequest.builder() .layoutParsingType(type) - .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority, dossier.getDossierTemplateId())) + .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority)) .originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) .imagesFileStorageId(optionalImageFileId) .tablesFileStorageId(optionalTableFileId) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java index 4e678e591..5eccf6b44 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestIdentifierService.java @@ -9,7 +9,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. @Service public class LayoutParsingRequestIdentifierService { - private enum IdentifierNames { DOSSIER_ID, FILE_ID, @@ -18,12 +17,6 @@ public class LayoutParsingRequestIdentifierService { } - public String parseDossierTemplateId(Map identifiers) { - - return identifiers.get(IdentifierNames.DOSSIER_TEMPLATE_ID.name()); - } - - public String parseDossierId(Map identifiers) { return identifiers.get(IdentifierNames.DOSSIER_ID.name()); @@ -42,16 +35,9 @@ public class LayoutParsingRequestIdentifierService { } - public Map buildIdentifier(String dossierId, String fileId, boolean priority, String dossierTemplateId) { + public Map buildIdentifier(String dossierId, String fileId, boolean priority) { - return Map.of(IdentifierNames.DOSSIER_TEMPLATE_ID.name(), - dossierTemplateId, - IdentifierNames.DOSSIER_ID.name(), - dossierId, - IdentifierNames.FILE_ID.name(), - fileId, - IdentifierNames.PRIORITY.name(), - String.valueOf(priority)); + return Map.of(IdentifierNames.DOSSIER_ID.name(), dossierId, IdentifierNames.FILE_ID.name(), fileId, IdentifierNames.PRIORITY.name(), String.valueOf(priority)); } } -- 2.47.2 From 93d2f41720500def2d9e6d0327628b203839265b Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 31 Jul 2023 14:21:42 +0200 Subject: [PATCH 06/11] use final version --- persistence-service-v1/persistence-service-processor-v1/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 056486ba5..191f8c8c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - RED-6725-2 + 0.14.0 -- 2.47.2 From e794a8effd71d3b7d3bc9315a7d12effecfb41d6 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 31 Jul 2023 14:33:48 +0200 Subject: [PATCH 07/11] add application type --- .../src/main/resources/application.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml index f0879bc2e..b3ade894a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml @@ -7,6 +7,9 @@ redaction-report-service.url: "http://redaction-report-service-v1:8080" search-service.url: "http://search-service-v1:8080" tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal" +application: + type: "RedactManager" + server: port: 8080 -- 2.47.2 From b7d51a07015011940109ac39a1184aa5b8cd5046 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 31 Jul 2023 15:36:45 +0200 Subject: [PATCH 08/11] add simplifiedText --- persistence-service-v1/persistence-service-processor-v1/pom.xml | 2 +- .../service/layoutparsing/LayoutParsingRequestFactory.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 191f8c8c9..4e6b8e389 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.14.0 + 0.16.0 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index 5c268160d..92cbab720 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -50,6 +50,7 @@ public class LayoutParsingRequestFactory { .structureFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) .textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) .positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)) + .simplifiedTextStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT)) .build(); } -- 2.47.2 From a6cf632d6e5a3d625809619e4d2cc3f4b52da5c5 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 1 Aug 2023 00:07:30 +0200 Subject: [PATCH 09/11] fix some log messages --- .../persistence-service-processor-v1/pom.xml | 2 +- .../v1/processor/service/FileManagementStorageService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 4e6b8e389..ffc4021b0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.16.0 + 0.17.0 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 81ca3f41d..0571a3abf 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -64,7 +64,7 @@ public class FileManagementStorageService { try { return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class); } catch (StorageObjectDoesNotExist e) { - log.debug("Text not available."); + log.debug("RedactionLog does not exist"); throw new NotFoundException("RedactionLog does not exist"); } } @@ -76,7 +76,7 @@ public class FileManagementStorageService { return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class); } catch (StorageObjectDoesNotExist e) { log.debug("SectionGrid not available."); - throw new NotFoundException("RedactionLog does not exist"); + throw new NotFoundException("SectionGrid does not exist"); } catch (Exception e) { throw new RuntimeException("Could not convert SectionGrid", e); } -- 2.47.2 From 7e4a034784142a91b5208700f3e8710a8098225c Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 1 Aug 2023 00:45:46 +0200 Subject: [PATCH 10/11] add sectio grid id --- .../service/layoutparsing/LayoutParsingRequestFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index 92cbab720..ea41d4def 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -51,6 +51,7 @@ public class LayoutParsingRequestFactory { .textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) .positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)) .simplifiedTextStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT)) + .sectionGridStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)) .build(); } -- 2.47.2 From 3fae3c0627a8c48d4063dfa7db52becbff4652ca Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 1 Aug 2023 10:44:59 +0200 Subject: [PATCH 11/11] update layoutparser version --- .../persistence-service-processor-v1/pom.xml | 2 +- .../queue/LayoutParsingFinishedMessageReceiver.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index ffc4021b0..42549b2fa 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.17.0 + 0.19.0 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 4313d11a8..7f4772ae3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -48,14 +49,14 @@ public class LayoutParsingFinishedMessageReceiver { @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_DLQ) public void handleDLQMessage(Message failedMessage) { - var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), AnalyzeRequest.class); + var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class); log.info("Failed to process analyze request: {}", analyzeRequest); String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER); OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER); timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp); - fileStatusProcessingUpdateService.analysisFailed(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), + fileStatusProcessingUpdateService.analysisFailed(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()), + layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()), new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp)); } -- 2.47.2