RED-7375: integrate table extractor
This commit is contained in:
Kilian Schuettler
parent
07c5530631
commit
8a6dd4c5e8
@ -59,6 +59,10 @@ public class MessagingConfiguration {
|
||||
public static final String CV_ANALYSIS_RESPONSE_QUEUE = "cv_analysis_response_queue";
|
||||
public static final String CV_ANALYSIS_DLQ = "cv_analysis_dead_letter_queue";
|
||||
|
||||
public static final String TABLE_EXTRACTOR_QUEUE = "table_extractor_request_queue";
|
||||
public static final String TABLE_EXTRACTOR_RESPONSE_QUEUE = "table_extractor_response_queue";
|
||||
public static final String TABLE_EXTRACTOR_DLQ = "table_extractor_dead_letter_queue";
|
||||
|
||||
public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";
|
||||
public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql";
|
||||
|
||||
@ -338,4 +342,26 @@ public class MessagingConfiguration {
|
||||
|
||||
return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue tableExtractorRequestQueue() {
|
||||
|
||||
return QueueBuilder.durable(TABLE_EXTRACTOR_QUEUE)//
|
||||
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue tableExtractorResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(TABLE_EXTRACTOR_RESPONSE_QUEUE)//
|
||||
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue tableExtractorDLQ() {
|
||||
|
||||
return QueueBuilder.durable(TABLE_EXTRACTOR_DLQ).build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class TableExtractorRequest {
|
||||
|
||||
public static final String TABLE_EXTRACTOR_FILE_EXTENSION = "EXTRACTED_TABLES.json.gz";
|
||||
|
||||
public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz";
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private String targetFileExtension;
|
||||
private String responseFileExtension;
|
||||
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class TableExtractorResponse {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
|
||||
}
|
||||
@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Inter
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.TableExtractorRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
|
||||
@ -175,6 +176,12 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (settings.isTableExtractorEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.INVISIBLE_TABLES)) {
|
||||
log.info("Add file: {} from dossier {} to Table Extractor queue", fileId, dossierId);
|
||||
addToTableExtractorQueue(dossierId, fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
|
||||
|
||||
@ -230,6 +237,24 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
private void addToTableExtractorQueue(String dossierId, String fileId) {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.TABLE_EXTRACTOR_ANALYZING);
|
||||
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.TABLE_EXTRACTOR_QUEUE,
|
||||
TableExtractorRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.targetFileExtension(TableExtractorRequest.TARGET_FILE_EXTENSION)
|
||||
.responseFileExtension(TableExtractorRequest.TABLE_EXTRACTOR_FILE_EXTENSION)
|
||||
.build(),
|
||||
message -> {
|
||||
message.getMessageProperties().setPriority(1);
|
||||
return message;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void addToPreprocessingQueue(String dossierId, String fileId, String filename) {
|
||||
|
||||
|
||||
@ -0,0 +1,55 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.TableExtractorResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TableExtractorMessageReceiver {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final FileStatusService fileStatusService;
|
||||
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = MessagingConfiguration.TABLE_EXTRACTOR_RESPONSE_QUEUE)
|
||||
public void receive(TableExtractorResponse response) {
|
||||
|
||||
fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false);
|
||||
|
||||
log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.TABLE_EXTRACTOR_RESPONSE_QUEUE, response.getDossierId(), response.getFileId());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = MessagingConfiguration.TABLE_EXTRACTOR_DLQ)
|
||||
public void handleDLQMessage(Message failedMessage) {
|
||||
|
||||
var response = objectMapper.readValue(failedMessage.getBody(), TableExtractorResponse.class);
|
||||
|
||||
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.TABLE_EXTRACTOR_DLQ, response.getDossierId(), response.getFileId());
|
||||
fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(),
|
||||
response.getFileId(),
|
||||
new FileErrorInfo("table extractor failed", MessagingConfiguration.TABLE_EXTRACTOR_DLQ, "table-extractor", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -24,6 +24,7 @@ public class FileManagementServiceSettings {
|
||||
|
||||
private boolean imageServiceEnabled = true;
|
||||
private boolean nerServiceEnabled = true;
|
||||
private boolean tableExtractorEnabled = true;
|
||||
|
||||
private boolean storeImageFile = true;
|
||||
|
||||
|
||||
@ -30,6 +30,7 @@ cors.enabled: true
|
||||
persistence-service:
|
||||
imageServiceEnabled: false
|
||||
nerServiceEnabled: false
|
||||
tableExtractorEnabled: false
|
||||
storeImageFile: false
|
||||
applicationName: RedactManager
|
||||
fforesight:
|
||||
|
||||
@ -17,6 +17,7 @@ public enum FileType {
|
||||
TEXT_HIGHLIGHTS(".json"),
|
||||
FIGURE(".json"),
|
||||
TABLES(".json"),
|
||||
INVISIBLE_TABLES(".json"),
|
||||
COMPONENTS(".json"),
|
||||
// document is split into 4 files, all should be overridden/deleted at the same time
|
||||
DOCUMENT_TEXT(".json"),
|
||||
|
||||
@ -18,5 +18,6 @@ public enum ProcessingStatus {
|
||||
PRE_PROCESSING,
|
||||
PRE_PROCESSED,
|
||||
FIGURE_DETECTION_ANALYZING,
|
||||
TABLE_PARSING_ANALYZING
|
||||
TABLE_PARSING_ANALYZING,
|
||||
TABLE_EXTRACTOR_ANALYZING,
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user