diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index f635cef86..4b3ba01f9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -91,7 +91,7 @@ com.knecon.fforesight layoutparser-service-internal-api - 0.57.0 + yannik-0 diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java index 2d6bc486e..65880198a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java @@ -59,12 +59,17 @@ public class MessagingConfiguration { public static final String CV_ANALYSIS_RESPONSE_QUEUE = "cv_analysis_response_queue"; public static final String CV_ANALYSIS_DLQ = "cv_analysis_dead_letter_queue"; + public static final String TABLE_EXTRACTOR_QUEUE = "table_extractor_request_queue"; + public static final String TABLE_EXTRACTOR_RESPONSE_QUEUE = "table_extractor_response_queue"; + public static final String TABLE_EXTRACTOR_DLQ = "table_extractor_dead_letter_queue"; + public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue"; public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql"; public static final String X_ERROR_INFO_HEADER = "x-error-message"; public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp"; + @Bean public Queue nerRequestQueue() { @@ -317,19 +322,21 @@ public class MessagingConfiguration { .build(); } + @Bean public Queue layoutparsingRequestQueue() { - return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)// - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); } @Bean public Queue layoutparsingResponseQueue() { - return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ) + .build(); } @@ -338,4 +345,29 @@ public class MessagingConfiguration { return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build(); } + + + @Bean + public Queue tableExtractorRequestQueue() { + + return QueueBuilder.durable(TABLE_EXTRACTOR_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", TABLE_EXTRACTOR_DLQ).build(); + } + + + @Bean + public Queue tableExtractorResponseQueue() { + + return QueueBuilder.durable(TABLE_EXTRACTOR_RESPONSE_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", TABLE_EXTRACTOR_DLQ) + .build(); + } + + + @Bean + public Queue tableExtractorDLQ() { + + return QueueBuilder.durable(TABLE_EXTRACTOR_DLQ).build(); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorRequest.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorRequest.java new file mode 100644 index 000000000..6ec502b84 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorRequest.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class TableExtractorRequest { + + public static final String TABLE_EXTRACTOR_FILE_EXTENSION = FileType.EXTRACTED_TABLES.name() + FileType.EXTRACTED_TABLES.getExtension() + ".gz"; + + public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz"; + + private String dossierId; + private String fileId; + private String targetFileExtension; + private String responseFileExtension; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorResponse.java new file mode 100644 index 000000000..ffec256d7 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/TableExtractorResponse.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class TableExtractorResponse { + + private String dossierId; + private String fileId; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 24912b848..ef9854c23 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Inter import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; +import com.iqser.red.service.persistence.management.v1.processor.model.TableExtractorRequest; import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; @@ -175,6 +176,12 @@ public class FileStatusService { return; } + if (settings.isTableExtractorEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.EXTRACTED_TABLES)) { + log.info("Add file: {} from dossier {} to Table Extractor queue", fileId, dossierId); + addToTableExtractorQueue(dossierId, fileId); + return; + } + var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper()); fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); @@ -230,6 +237,24 @@ public class FileStatusService { } + private void addToTableExtractorQueue(String dossierId, String fileId) { + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.TABLE_EXTRACTOR_ANALYZING); + + rabbitTemplate.convertAndSend(MessagingConfiguration.TABLE_EXTRACTOR_QUEUE, + TableExtractorRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(TableExtractorRequest.TARGET_FILE_EXTENSION) + .responseFileExtension(TableExtractorRequest.TABLE_EXTRACTOR_FILE_EXTENSION) + .build(), + message -> { + message.getMessageProperties().setPriority(1); + return message; + }); + } + + @SneakyThrows public void addToPreprocessingQueue(String dossierId, String fileId, String filename) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java index 5936ed66d..451a56bd8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java @@ -39,18 +39,22 @@ public class LayoutParsingRequestFactory { Optional optionalTableFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.TABLES) // ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TABLES)) : Optional.empty(); + Optional optionalTableExtractorFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.EXTRACTED_TABLES) // + ? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.EXTRACTED_TABLES)) : Optional.empty(); + + return LayoutParsingRequest.builder() .layoutParsingType(type) .identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority)) .originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)) .imagesFileStorageId(optionalImageFileId) .tablesFileStorageId(optionalTableFileId) + .tableExtractorFileId(optionalTableExtractorFileId) .pageFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) .structureFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) .textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) .positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)) .simplifiedTextStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT)) - .sectionGridStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)) .viewerDocumentStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT)) .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/TableExtractorMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/TableExtractorMessageReceiver.java new file mode 100644 index 000000000..b53ee3a19 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/TableExtractorMessageReceiver.java @@ -0,0 +1,55 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.queue; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse; +import com.iqser.red.service.persistence.management.v1.processor.model.TableExtractorResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TableExtractorMessageReceiver { + + private final ObjectMapper objectMapper; + private final FileStatusService fileStatusService; + private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + + + @SneakyThrows + @RabbitListener(queues = MessagingConfiguration.TABLE_EXTRACTOR_RESPONSE_QUEUE) + public void receive(TableExtractorResponse response) { + + fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false); + + log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.TABLE_EXTRACTOR_RESPONSE_QUEUE, response.getDossierId(), response.getFileId()); + } + + + @SneakyThrows + @RabbitListener(queues = MessagingConfiguration.TABLE_EXTRACTOR_DLQ) + public void handleDLQMessage(Message failedMessage) { + + var response = objectMapper.readValue(failedMessage.getBody(), TableExtractorResponse.class); + + log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.TABLE_EXTRACTOR_DLQ, response.getDossierId(), response.getFileId()); + fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), + response.getFileId(), + new FileErrorInfo("table extractor failed", MessagingConfiguration.TABLE_EXTRACTOR_DLQ, "table-extractor", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java index 452013c4b..8d942f617 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/FileManagementServiceSettings.java @@ -24,6 +24,7 @@ public class FileManagementServiceSettings { private boolean imageServiceEnabled = true; private boolean nerServiceEnabled = true; + private boolean tableExtractorEnabled = true; private boolean storeImageFile = true; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml index c49a6fac7..bf4fa88f8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml @@ -30,6 +30,7 @@ cors.enabled: true persistence-service: imageServiceEnabled: false nerServiceEnabled: false + tableExtractorEnabled: false storeImageFile: false applicationName: RedactManager fforesight: diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml index c85f2c996..06807fcb7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml @@ -58,6 +58,7 @@ server: persistence-service: imageServiceEnabled: false + tableExtractorEnabled: false metrics: diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index 8008340ee..cea316b90 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -17,6 +17,7 @@ public enum FileType { TEXT_HIGHLIGHTS(".json"), FIGURE(".json"), TABLES(".json"), + EXTRACTED_TABLES(".json"), COMPONENTS(".json"), // document is split into 4 files, all should be overridden/deleted at the same time DOCUMENT_TEXT(".json"), diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java index f65ceabe1..1a158927a 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -18,5 +18,6 @@ public enum ProcessingStatus { PRE_PROCESSING, PRE_PROCESSED, FIGURE_DETECTION_ANALYZING, - TABLE_PARSING_ANALYZING + TABLE_PARSING_ANALYZING, + TABLE_EXTRACTOR_ANALYZING, }