RED-6725: integrate layoutparsing service
This commit is contained in:
Kilian Schuettler
parent
e7b29fb683
commit
12703c6eda
@ -88,6 +88,12 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.knecon.fforesight</groupId>
|
||||
<artifactId>layoutparser-service-internal-api</artifactId>
|
||||
<version>0.3.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>search-service-api-v1</artifactId>
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.configuration;
|
||||
|
||||
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
|
||||
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
|
||||
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@ -58,6 +62,8 @@ public class MessagingConfiguration {
|
||||
public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";
|
||||
public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql";
|
||||
|
||||
public static final String X_ERROR_INFO_HEADER = "x-error-message";
|
||||
public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
|
||||
|
||||
@Bean
|
||||
public Queue nerRequestQueue() {
|
||||
@ -311,4 +317,25 @@ public class MessagingConfiguration {
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue layoutparsingRequestQueue() {
|
||||
|
||||
return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
|
||||
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue layoutparsingResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
|
||||
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue layoutparsingDLQ() {
|
||||
|
||||
return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -40,12 +40,6 @@ public class FileStatusProcessingUpdateService {
|
||||
|
||||
switch (analyzeResult.getMessageType()) {
|
||||
|
||||
case STRUCTURE_ANALYSE:
|
||||
|
||||
//TODO This might be also priority depending on what was the pervious call.
|
||||
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
|
||||
break;
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
fileStatusService.setStatusProcessed(analyzeResult.getFileId());
|
||||
manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
|
||||
@ -75,7 +69,6 @@ public class FileStatusProcessingUpdateService {
|
||||
}
|
||||
|
||||
@Transactional
|
||||
|
||||
public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse) {
|
||||
|
||||
fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(), untouchedDocumentResponse.getFileSize());
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@ -12,7 +14,6 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
@ -22,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysi
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
@ -74,6 +76,7 @@ public class FileStatusService {
|
||||
private final ReanalysisRequiredStatusService reanalysisRequiredStatusService;
|
||||
private final ViewedPagesPersistenceService viewedPagesPersistenceService;
|
||||
private final FileManagementServiceSettings fileManagementServiceSettings;
|
||||
private final LayoutParsingRequestFactory layoutParsingRequestFactory;
|
||||
|
||||
|
||||
@Transactional
|
||||
@ -174,7 +177,7 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
var fileModel = MagicConverter.convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
|
||||
fileModel = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
|
||||
|
||||
var dossierTemplate = dossier.getDossierTemplate();
|
||||
if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null) {
|
||||
@ -183,22 +186,20 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
MessageType messageType = null;
|
||||
|
||||
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
|
||||
messageType = MessageType.STRUCTURE_ANALYSE;
|
||||
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) {
|
||||
var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority, dossier);
|
||||
rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
|
||||
return;
|
||||
}
|
||||
|
||||
if (messageType == null && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
|
||||
if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
|
||||
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
|
||||
addToNerQueue(dossierId, fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (messageType == null) {
|
||||
boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse;
|
||||
messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel);
|
||||
}
|
||||
boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse;
|
||||
MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel);
|
||||
|
||||
var analyseRequest = AnalyzeRequest.builder()
|
||||
.messageType(messageType)
|
||||
|
||||
@ -10,7 +10,6 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
|
||||
@ -39,7 +38,6 @@ public class ReanalysisService {
|
||||
private final IndexingService indexingService;
|
||||
private final PDFTronClient pDFTronRedactionClient;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
|
||||
public void reanalyzeDossier(String dossierId, boolean force) {
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class LayoutParsingRequestFactory {
|
||||
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService;
|
||||
|
||||
|
||||
public LayoutParsingRequest build(String dossierId, String fileId, boolean priority, DossierEntity dossier) {
|
||||
|
||||
Optional<String> optionalImageFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO) //
|
||||
? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)) : Optional.empty();
|
||||
|
||||
Optional<String> optionalTableFileId = fileManagementStorageService.objectExists(dossierId, fileId, FileType.TABLES) //
|
||||
? Optional.of(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TABLES)) : Optional.empty();
|
||||
|
||||
return LayoutParsingRequest.builder()
|
||||
.identifier(layoutParsingRequestIdentifierService.buildIdentifier(dossierId, fileId, priority, dossier.getDossierTemplateId()))
|
||||
.originFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN))
|
||||
.imagesFileStorageId(optionalImageFileId)
|
||||
.tablesFileStorageId(optionalTableFileId)
|
||||
.pageFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES))
|
||||
.structureFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE))
|
||||
.textBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT))
|
||||
.positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION))
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
|
||||
@Service
|
||||
public class LayoutParsingRequestIdentifierService {
|
||||
|
||||
|
||||
private enum IdentifierNames {
|
||||
DOSSIER_ID,
|
||||
FILE_ID,
|
||||
PRIORITY,
|
||||
DOSSIER_TEMPLATE_ID
|
||||
}
|
||||
|
||||
|
||||
public String parseDossierTemplateId(Map<String, String> identifiers) {
|
||||
|
||||
return identifiers.get(IdentifierNames.DOSSIER_TEMPLATE_ID.name());
|
||||
}
|
||||
|
||||
|
||||
public String parseDossierId(Map<String, String> identifiers) {
|
||||
|
||||
return identifiers.get(IdentifierNames.DOSSIER_ID.name());
|
||||
}
|
||||
|
||||
|
||||
public String parseFileId(Map<String, String> identifiers) {
|
||||
|
||||
return identifiers.get(IdentifierNames.FILE_ID.name());
|
||||
}
|
||||
|
||||
|
||||
public Boolean parsePriority(Map<String, String> identifiers) {
|
||||
|
||||
return Boolean.parseBoolean(identifiers.get(IdentifierNames.PRIORITY.name()));
|
||||
}
|
||||
|
||||
|
||||
public Map<String, String> buildIdentifier(String dossierId, String fileId, boolean priority, String dossierTemplateId) {
|
||||
|
||||
return Map.of(IdentifierNames.DOSSIER_TEMPLATE_ID.name(),
|
||||
dossierTemplateId,
|
||||
IdentifierNames.DOSSIER_ID.name(),
|
||||
dossierId,
|
||||
IdentifierNames.FILE_ID.name(),
|
||||
fileId,
|
||||
IdentifierNames.PRIORITY.name(),
|
||||
String.valueOf(priority));
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
@ -10,6 +10,8 @@ import org.springframework.stereotype.Service;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.OffsetDateTime;
|
||||
@ -13,6 +13,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
@ -0,0 +1,63 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class LayoutParsingFinishedMessageReceiver {
|
||||
|
||||
private final FileStatusService fileStatusService;
|
||||
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)
|
||||
public void receive(LayoutParsingFinishedEvent response) {
|
||||
|
||||
fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()),
|
||||
layoutParsingRequestIdentifierService.parseFileId(response.identifier()),
|
||||
layoutParsingRequestIdentifierService.parsePriority(response.identifier()));
|
||||
|
||||
log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_DLQ)
|
||||
public void handleDLQMessage(Message failedMessage) {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), AnalyzeRequest.class);
|
||||
log.info("Failed to process analyze request: {}", analyzeRequest);
|
||||
String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER);
|
||||
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
|
||||
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp);
|
||||
fileStatusProcessingUpdateService.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.OffsetDateTime;
|
||||
@ -12,6 +12,8 @@ import org.springframework.stereotype.Service;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
@ -7,6 +7,7 @@ import org.springframework.stereotype.Service;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
@ -0,0 +1,205 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.FileModelMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
|
||||
class ReanalysisServiceTest {
|
||||
|
||||
private ReanalysisService reanalysisService;
|
||||
|
||||
private FileStatusService fileStatusService;
|
||||
|
||||
@Mock
|
||||
private ReanalysisRequiredStatusService reanalysisRequiredStatusService;
|
||||
|
||||
@Mock
|
||||
private IndexingService indexingService;
|
||||
@Mock
|
||||
private DossierPersistenceService dossierPersistenceService;
|
||||
@Mock
|
||||
private PDFTronClient pdfTronClient;
|
||||
@Mock
|
||||
private RabbitTemplate rabbitTemplate;
|
||||
@Mock
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
@Mock
|
||||
private ObjectMapper objectMapper;
|
||||
@Mock
|
||||
private ManualRedactionProviderService manualRedactionProviderService;
|
||||
@Mock
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
@Mock
|
||||
private LegalBasisChangePersistenceService legalBasisChangePersistenceService;
|
||||
@Mock
|
||||
private ImageRecategorizationPersistenceService imageRecategorizationPersistenceService;
|
||||
@Mock
|
||||
private CommentPersistenceService commentPersistenceService;
|
||||
@Mock
|
||||
private ForceRedactionPersistenceService forceRedactionPersistenceService;
|
||||
@Mock
|
||||
private RemoveRedactionPersistenceService removeRedactionPersistenceService;
|
||||
@Mock
|
||||
private AddRedactionPersistenceService addRedactionPersistenceService;
|
||||
@Mock
|
||||
private ResizeRedactionPersistenceService resizeRedactionPersistenceService;
|
||||
@Mock
|
||||
private FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
|
||||
@Mock
|
||||
private FileManagementServiceSettings settings;
|
||||
@Mock
|
||||
private ViewedPagesPersistenceService viewedPagesPersistenceService;
|
||||
@Mock
|
||||
private FileManagementServiceSettings fileManagementServiceSettings;
|
||||
@Mock
|
||||
private DictionaryPersistenceService dictionaryPersistenceService;
|
||||
@Mock
|
||||
private RulesPersistenceService rulesPersistenceService;
|
||||
@Mock
|
||||
private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||
|
||||
private static String DOSSIER_ID = "123";
|
||||
private static String FILE_ID = "456";
|
||||
|
||||
private FileModel fileModel;
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void stubAndCreate() {
|
||||
|
||||
fileModel = MagicConverter.convert(getTestFileEntity(), FileModel.class, new FileModelMapper());
|
||||
MockitoAnnotations.openMocks(this);
|
||||
|
||||
fileStatusService = new FileStatusService(fileStatusPersistenceService,
|
||||
dossierPersistenceService,
|
||||
rabbitTemplate,
|
||||
ObjectMapperFactory.create(),
|
||||
manualRedactionProviderService,
|
||||
fileManagementStorageService,
|
||||
legalBasisChangePersistenceService,
|
||||
imageRecategorizationPersistenceService,
|
||||
commentPersistenceService,
|
||||
forceRedactionPersistenceService,
|
||||
removeRedactionPersistenceService,
|
||||
addRedactionPersistenceService,
|
||||
resizeRedactionPersistenceService,
|
||||
fileAttributeConfigPersistenceService,
|
||||
settings,
|
||||
reanalysisRequiredStatusService,
|
||||
viewedPagesPersistenceService,
|
||||
fileManagementServiceSettings);
|
||||
reanalysisService = new ReanalysisService(fileStatusService, dossierPersistenceService, indexingService, pdfTronClient, fileManagementStorageService);
|
||||
|
||||
when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(getTestFileEntity());
|
||||
when(dossierPersistenceService.getAndValidateDossier(DOSSIER_ID)).thenReturn(getDossierEntity());
|
||||
when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.ORIGIN)).thenReturn(true);
|
||||
when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.TEXT)).thenReturn(true);
|
||||
when(fileManagementStorageService.objectExists(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES)).thenReturn(true);
|
||||
when(settings.isFigureDetectionEnabled()).thenReturn(false);
|
||||
when(settings.isCvTableParsingEnabled()).thenReturn(false);
|
||||
when(settings.isImageServiceEnabled()).thenReturn(false);
|
||||
when(settings.isOcrByDefault()).thenReturn(false);
|
||||
when(dossierPersistenceService.findByDossierId(DOSSIER_ID)).thenReturn(getDossierEntity());
|
||||
when(reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true)) //
|
||||
.thenReturn(fileModel);
|
||||
}
|
||||
|
||||
|
||||
private Dossier getTestDossier() {
|
||||
|
||||
return Dossier.builder().id(DOSSIER_ID).build();
|
||||
}
|
||||
|
||||
|
||||
private static FileStatus getTestFileStatus() {
|
||||
|
||||
return FileStatus.builder().fileId(FILE_ID).dossierId(DOSSIER_ID).processingStatus(ProcessingStatus.PROCESSED).build();
|
||||
}
|
||||
|
||||
|
||||
private static FileEntity getTestFileEntity() {
|
||||
|
||||
return FileEntity.builder()
|
||||
.id(FILE_ID)
|
||||
.dossierId(DOSSIER_ID)
|
||||
.workflowStatus(WorkflowStatus.NEW)
|
||||
.fileAttributes(Collections.emptyList())
|
||||
.excluded(false)
|
||||
.processingStatus(ProcessingStatus.PROCESSED)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static DossierEntity getDossierEntity() {
|
||||
|
||||
return DossierEntity.builder().id(DOSSIER_ID).build();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void reanalysisTriggeredByHighLightConversionTest() {
|
||||
|
||||
var dossier = getTestDossier();
|
||||
var fileStatus = getTestFileStatus();
|
||||
|
||||
var textHighlightRequest = new TextHighlightConversionRequest(dossier.getDossierId(),
|
||||
fileStatus.getFileId(),
|
||||
Set.of("1", "2", "3"),
|
||||
TextHighlightConversionOperation.CONVERT);
|
||||
when(pdfTronClient.convertTextHighlights(textHighlightRequest)).thenReturn(true);
|
||||
|
||||
reanalysisService.convertTextHighlights(textHighlightRequest);
|
||||
assertThat(fileModel.getAnalysisVersion()).isEqualTo(1);
|
||||
verify(rabbitTemplate, times(1)).convertAndSend((String) any(), (Object) any());
|
||||
verify(fileStatusPersistenceService, times(1)).updateProcessingStatus(FILE_ID, ProcessingStatus.FULL_PROCESSING);
|
||||
}
|
||||
|
||||
}
|
||||
@ -135,20 +135,23 @@ public class FileStatus {
|
||||
private boolean hasHighlights;
|
||||
@Schema(description = "Size of the optimized, internally stored file.")
|
||||
private Long fileSize;
|
||||
@Schema(description = "Analysis Version.")
|
||||
@Schema(description = "Analysis Version.")
|
||||
private int analysisVersion;
|
||||
@Schema(description = "Last time the file was indexed in ES.")
|
||||
private OffsetDateTime lastIndexed;
|
||||
@Schema(description = "The error information for the error state of the file")
|
||||
private FileErrorInfo fileErrorInfo;
|
||||
|
||||
|
||||
@Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.")
|
||||
public OffsetDateTime getLastOCRTime() {
|
||||
|
||||
return ocrEndTime != null ? ocrEndTime : ocrStartTime;
|
||||
}
|
||||
|
||||
public String getId(){
|
||||
|
||||
public String getId() {
|
||||
|
||||
return fileId;
|
||||
}
|
||||
|
||||
|
||||
@ -9,14 +9,18 @@ public enum FileType {
|
||||
REDACTION_LOG(".json"),
|
||||
SIMPLIFIED_TEXT(".json"),
|
||||
SECTION_GRID(".json"),
|
||||
TEXT(".json"),
|
||||
TEXT(".json"), // TODO: refactor this away
|
||||
NER_ENTITIES(".json"),
|
||||
IMAGE_INFO(".json"),
|
||||
IMPORTED_REDACTIONS(".json"),
|
||||
TEXT_HIGHLIGHTS(".json"),
|
||||
FIGURE(".json"),
|
||||
TABLES(".json"),
|
||||
COMPONENTS(".json");
|
||||
COMPONENTS(".json"),
|
||||
DOCUMENT_TEXT(".json"),
|
||||
DOCUMENT_STRUCTURE(".json"),
|
||||
DOCUMENT_POSITION(".json"),
|
||||
DOCUMENT_PAGES(".json");
|
||||
|
||||
@Getter
|
||||
private final String extension;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user