RED-3350 Communication with entity recognition via queues
This commit is contained in:
Philipp Schramm
parent
6e62fc0479
commit
ffd4da35ad
@ -1,5 +1,17 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
@ -10,17 +22,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ -54,13 +57,12 @@ public class FileStatusPersistenceService {
|
||||
public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
|
||||
long legalBasisVersion, long duration, long dossierDictionaryVersion,
|
||||
int analysisVersion, int analysisNumber) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED,
|
||||
dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion,
|
||||
analysisVersion, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
|
||||
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
|
||||
}
|
||||
|
||||
|
||||
@ -75,14 +77,21 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
private boolean isFileDeleted(String fileId) {
|
||||
|
||||
return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||
approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null, approval);
|
||||
fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), approval ? OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS) : null, approval);
|
||||
|
||||
}
|
||||
|
||||
@ -95,26 +104,15 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
if (processingStatus == ProcessingStatus.PROCESSED) {
|
||||
// In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
} else {
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setUpdateStatusIndexingSuccessful(String fileId) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateLastOCRTime(String fileId, OffsetDateTime time) {
|
||||
|
||||
@ -146,14 +144,13 @@ public class FileStatusPersistenceService {
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
|
||||
boolean hasSuggestions) {
|
||||
public void setUpdateStatusIndexingSuccessful(String fileId) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
|
||||
fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
|
||||
|
||||
@ -178,8 +175,28 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
|
||||
boolean hasSuggestions) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
|
||||
}
|
||||
|
||||
|
||||
public FileEntity getStatus(String fileId) {
|
||||
|
||||
return fileRepository.findById(fileId).orElseThrow(() -> new NotFoundException("Unknown file=" + fileId));
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setExcludedPages(String fileId, Set<Integer> excludedPages) {
|
||||
|
||||
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
|
||||
if (file.isSoftOrHardDeleted()) {
|
||||
return;
|
||||
@ -195,18 +212,6 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
public FileEntity getStatus(String fileId) {
|
||||
|
||||
return fileRepository.findById(fileId).orElseThrow(() -> new NotFoundException("Unknown file=" + fileId));
|
||||
|
||||
}
|
||||
|
||||
|
||||
public List<FileEntity> getActiveFiles(String dossierId) {
|
||||
return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
public List<FileEntity> getStatusesForDossier(String dossierId) {
|
||||
|
||||
return fileRepository.findByDossierId(dossierId);
|
||||
@ -219,14 +224,12 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
|
||||
public List<FileEntity> getActiveFiles(String dossierId) {
|
||||
|
||||
int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
return fileRepository.findByDossierId(dossierId)
|
||||
.stream()
|
||||
.filter(f -> !f.isSoftOrHardDeleted())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@ -243,6 +246,17 @@ public class FileStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
|
||||
|
||||
int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void undelete(String fileId) {
|
||||
|
||||
@ -253,8 +267,8 @@ public class FileStatusPersistenceService {
|
||||
}, () -> {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
});
|
||||
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
|
||||
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), null);
|
||||
}
|
||||
|
||||
|
||||
@ -264,8 +278,8 @@ public class FileStatusPersistenceService {
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
if (updateCount == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -274,21 +288,26 @@ public class FileStatusPersistenceService {
|
||||
|
||||
@Transactional
|
||||
public void toggleExclusion(String fileId, boolean excluded) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS));
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
@ -302,22 +321,14 @@ public class FileStatusPersistenceService {
|
||||
file.setExcludedPages(new HashSet<>());
|
||||
});
|
||||
|
||||
int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS,
|
||||
WorkflowStatus.NEW,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now()
|
||||
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
if (countUpdate == 0) {
|
||||
throw new NotFoundException("Unknown file=" + fileId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime changeDate) {
|
||||
fileRepository.setLastAnnotationModificationDateForFile(fileId, changeDate);
|
||||
}
|
||||
|
||||
|
||||
private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file,
|
||||
Map<String, String> fileAttributesMap) {
|
||||
|
||||
@ -337,15 +348,23 @@ public class FileStatusPersistenceService {
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private boolean isFileDeleted(String fileId) {
|
||||
return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
|
||||
|
||||
@Transactional
|
||||
public void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime changeDate) {
|
||||
|
||||
fileRepository.setLastAnnotationModificationDateForFile(fileId, changeDate);
|
||||
}
|
||||
|
||||
|
||||
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
|
||||
|
||||
return fileRepository.existsByDossierIdAndLastUpdatedIsAfter(dossierId, since);
|
||||
}
|
||||
|
||||
|
||||
public int countSoftDeletedFiles(String dossierId) {
|
||||
|
||||
return fileRepository.countSoftDeletedFiles(dossierId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
package com.iqser.red.service.peristence.v1.server.configuration;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class MessagingConfiguration {
|
||||
@ -35,6 +36,37 @@ public class MessagingConfiguration {
|
||||
public static final String IMAGE_SERVICE_RESPONSE_QUEUE = "image_response_queue";
|
||||
public static final String IMAGE_SERVICE_DLQ = "image_dead_letter_queue";
|
||||
|
||||
public static final String NER_SERVICE_QUEUE = "entity_request_queue";
|
||||
public static final String NER_SERVICE_RESPONSE_QUEUE = "entity_response_queue";
|
||||
public static final String NER_SERVICE_DLQ = "entity_dead_letter_queue";
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue nerRequestQueue() {
|
||||
|
||||
return QueueBuilder.durable(NER_SERVICE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue nerResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue nerResponseDLQ() {
|
||||
|
||||
return QueueBuilder.durable(NER_SERVICE_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue imageRequestQueue() {
|
||||
@ -55,6 +87,7 @@ public class MessagingConfiguration {
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue imageResponseDLQ() {
|
||||
|
||||
@ -62,7 +95,6 @@ public class MessagingConfiguration {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionQueue() {
|
||||
|
||||
|
||||
@ -1,13 +1,17 @@
|
||||
package com.iqser.red.service.peristence.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class FileStatusProcessingUpdateController implements FileStatusProcessingUpdateResource {
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.peristence.v1.server.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class NerServiceRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
|
||||
}
|
||||
@ -1,5 +1,12 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
@ -8,15 +15,10 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -49,6 +51,7 @@ public class FileManagementStorageService {
|
||||
|
||||
|
||||
public void storeObject(String storageId, InputStream data) {
|
||||
|
||||
storageService.storeObject(storageId, data);
|
||||
}
|
||||
|
||||
@ -81,11 +84,25 @@ public class FileManagementStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
public boolean imageInfoExists(String dossierId, String fileId){
|
||||
|
||||
public boolean imageInfoExists(String dossierId, String fileId) {
|
||||
|
||||
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
|
||||
}
|
||||
|
||||
|
||||
public boolean textExists(String dossierId, String fileId) {
|
||||
|
||||
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
|
||||
}
|
||||
|
||||
|
||||
public boolean nerEntitiesExists(String dossierId, String fileId) {
|
||||
|
||||
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
|
||||
}
|
||||
|
||||
|
||||
public void deleteObject(String dossierId, String fileId, FileType fileType) {
|
||||
|
||||
storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType));
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import com.iqser.red.service.search.v1.model.IndexMessageType;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
import com.iqser.red.service.search.v1.model.IndexMessageType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -21,6 +23,8 @@ public class FileStatusProcessingUpdateService {
|
||||
private final DossierPersistenceService dossierPersistenceService;
|
||||
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
|
||||
private final ManualRedactionService manualRedactionService;
|
||||
private final FileManagementServiceSettings settings;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
|
||||
public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
|
||||
@ -29,16 +33,25 @@ public class FileStatusProcessingUpdateService {
|
||||
|
||||
switch (analyzeResult.getMessageType()) {
|
||||
|
||||
case STRUCTURE_ANALYSE:
|
||||
if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
|
||||
fileStatusService.addToNerQueue(dossierId, fileId);
|
||||
} else {
|
||||
analyzeResult.setMessageType(MessageType.ANALYSE);
|
||||
analysisSuccessful(dossierId, fileId, analyzeResult);
|
||||
}
|
||||
break;
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
fileStatusService.setStatusProcessed(analyzeResult.getFileId());
|
||||
manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
|
||||
break;
|
||||
|
||||
case ANALYSE:
|
||||
case REANALYSE:
|
||||
case FULL_ANALYSE:
|
||||
default:
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
|
||||
.getRetryCount());
|
||||
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
|
||||
fileStatusService.setStatusSuccessful(fileId, analyzeResult);
|
||||
return null;
|
||||
});
|
||||
@ -57,8 +70,7 @@ public class FileStatusProcessingUpdateService {
|
||||
public void ocrSuccessful(String dossierId, String fileId) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
|
||||
.getRetryCount());
|
||||
log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
|
||||
|
||||
fileStatusService.updateLastOCRTime(fileId);
|
||||
fileStatusService.setStatusFullReprocess(dossierId, fileId, 2);
|
||||
@ -103,8 +115,7 @@ public class FileStatusProcessingUpdateService {
|
||||
public void indexingSuccessful(String dossierId, String fileId) {
|
||||
|
||||
retryTemplate.execute(retryContext -> {
|
||||
log.info("Indexing Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext
|
||||
.getRetryCount());
|
||||
log.info("Indexing Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
|
||||
fileStatusService.setStatusIndexingSuccessful(fileId);
|
||||
return null;
|
||||
});
|
||||
|
||||
@ -1,36 +1,47 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.peristence.v1.server.controller.RulesController;
|
||||
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -98,11 +109,30 @@ public class FileStatusService {
|
||||
|
||||
public void setStatusSuccessful(String fileId, AnalyzeResult analyzeResult) {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(),
|
||||
analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(),
|
||||
analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(),
|
||||
analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(),
|
||||
analyzeResult.getAnalysisNumber());
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), analyzeResult.getAnalysisNumber());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse,
|
||||
boolean triggeredManually) {
|
||||
|
||||
log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually) {
|
||||
log.debug("File {} is excluded from automatic analysis", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
|
||||
}
|
||||
|
||||
|
||||
@ -118,63 +148,6 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void createStatus(String dossierId, String fileId, String uploader, String filename) {
|
||||
|
||||
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
|
||||
addToAnalysisQueue(dossierId, fileId, 1, Set.of());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority) {
|
||||
|
||||
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) {
|
||||
|
||||
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
|
||||
|
||||
log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
if(fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually){
|
||||
log.debug("File {} is excluded from automatic analysis", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
|
||||
}
|
||||
|
||||
|
||||
public void setStatusOcrProcessing(String dossierId, String fileId) {
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
|
||||
addToOcrQueue(dossierId, fileId, 2);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusFullReprocess(String dossierId, String fileId, int priority) {
|
||||
|
||||
@ -186,10 +159,119 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
|
||||
|
||||
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
|
||||
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
@SuppressWarnings("PMD")
|
||||
protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
|
||||
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
var fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
boolean reanalyse = isReanalyse(dossier, fileStatus);
|
||||
|
||||
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
|
||||
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
|
||||
addToImageQueue(dossierId, fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
|
||||
|
||||
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
|
||||
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
|
||||
addToNerQueue(dossierId, fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
var analyseRequest = AnalyzeRequest.builder()
|
||||
.messageType(messageType)
|
||||
.dossierId(dossierId)
|
||||
.analysisNumber(fileStatus.getNumberOfAnalyses() + 1)
|
||||
.sectionsToReanalyse(sectionsToReanalyse)
|
||||
.fileId(fileId)
|
||||
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
|
||||
.dossierTemplateId(dossier.getDossierTemplateId())
|
||||
.lastProcessed(fileStatus.getLastProcessed())
|
||||
.fileAttributes(convert(fileStatus.getFileAttributes()))
|
||||
.excludedPages(fileStatus.getExcludedPages())
|
||||
.build();
|
||||
|
||||
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
|
||||
|
||||
setStatusProcessing(fileId);
|
||||
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
|
||||
message.getMessageProperties().setPriority(priority);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) {
|
||||
|
||||
return !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
|
||||
.isAfter(fileStatus.getLastFileAttributeChange()));
|
||||
}
|
||||
|
||||
|
||||
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
|
||||
ProcessingStatus processingStatus) {
|
||||
|
||||
if (!fileManagementStorageService.textExists(dossierId, fileId)) {
|
||||
return MessageType.STRUCTURE_ANALYSE;
|
||||
}
|
||||
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
if (reanalyse) {
|
||||
return MessageType.REANALYSE;
|
||||
}
|
||||
return MessageType.ANALYSE;
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority) {
|
||||
|
||||
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) {
|
||||
|
||||
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void createStatus(String dossierId, String fileId, String uploader, String filename) {
|
||||
|
||||
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
|
||||
addToAnalysisQueue(dossierId, fileId, 1, Set.of());
|
||||
}
|
||||
|
||||
|
||||
public void setStatusIndexingSuccessful(String fileId) {
|
||||
|
||||
fileStatusPersistenceService.setUpdateStatusIndexingSuccessful(fileId);
|
||||
@ -259,53 +341,23 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
@SuppressWarnings("PMD")
|
||||
protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
|
||||
public void setStatusOcrProcessing(String dossierId, String fileId) {
|
||||
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
var fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
log.debug("File {} is excluded", fileId);
|
||||
return;
|
||||
}
|
||||
|
||||
var fileAttributes = fileStatus.getFileAttributes();
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
|
||||
addToOcrQueue(dossierId, fileId, 2);
|
||||
}
|
||||
|
||||
boolean reanalyse = !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
|
||||
.isAfter(fileStatus.getLastFileAttributeChange()));
|
||||
|
||||
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
|
||||
addToImageQueue(dossierId, fileId);
|
||||
return;
|
||||
}
|
||||
public void setStatusNerAnalyzing(String fileId) {
|
||||
|
||||
var analyseRequest = AnalyzeRequest.builder()
|
||||
.messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE)
|
||||
.dossierId(dossierId)
|
||||
.analysisNumber(fileStatus.getNumberOfAnalyses()+1)
|
||||
.sectionsToReanalyse(sectionsToReanalyse)
|
||||
.fileId(fileId)
|
||||
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
|
||||
.dossierTemplateId(dossier.getDossierTemplateId())
|
||||
.lastProcessed(fileStatus.getLastProcessed())
|
||||
.fileAttributes(convert(fileAttributes))
|
||||
.excludedPages(fileStatus.getExcludedPages())
|
||||
.build();
|
||||
|
||||
setStatusProcessing(fileId);
|
||||
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
|
||||
message.getMessageProperties().setPriority(priority);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING);
|
||||
}
|
||||
|
||||
|
||||
@ -427,6 +479,35 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void setStatusAnalyse(String dossierId, String fileId, int priority) {
|
||||
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
|
||||
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
|
||||
}
|
||||
|
||||
|
||||
protected void addToNerQueue(String dossierId, String fileId) {
|
||||
|
||||
setStatusNerAnalyzing(fileId);
|
||||
try {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(new NerServiceRequest(dossierId, fileId)), message -> {
|
||||
message.getMessageProperties().setPriority(1);
|
||||
return message;
|
||||
});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int countSoftDeletedFiles(String dossierId) {
|
||||
|
||||
return fileStatusPersistenceService.countSoftDeletedFiles(dossierId);
|
||||
|
||||
@ -2,7 +2,6 @@ package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
@ -39,7 +38,7 @@ public class ImageMessageReceiver {
|
||||
String fileId = (String) imageResponse.get("fileId");
|
||||
|
||||
fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, in);
|
||||
fileStatusService.setStatusFullReprocess(dossierId, fileId, 1);
|
||||
fileStatusService.setStatusAnalyse(dossierId, fileId, 1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,55 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class NerMessageReceiver {
|
||||
|
||||
private final FileStatusService fileStatusService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE)
|
||||
public void receive(byte[] in) {
|
||||
|
||||
HashMap<String, Object> entityResponse = objectMapper.readValue(in, new TypeReference<>() {
|
||||
});
|
||||
|
||||
String dossierId = (String) entityResponse.get("dossierId");
|
||||
String fileId = (String) entityResponse.get("fileId");
|
||||
|
||||
log.info("Received NER Message from {} {}", dossierId, fileId);
|
||||
fileStatusService.addToAnalysisQueue(dossierId, fileId, 2, null);
|
||||
}
|
||||
|
||||
|
||||
@RabbitListener(queues = MessagingConfiguration.NER_SERVICE_DLQ)
|
||||
public void handleDLQMessage(Message failedMessage) throws IOException {
|
||||
|
||||
HashMap<String, Object> entityResponse = objectMapper.readValue(failedMessage.getBody(), new TypeReference<>() {
|
||||
});
|
||||
String dossierId = (String) entityResponse.get("dossierId");
|
||||
String fileId = (String) entityResponse.get("fileId");
|
||||
|
||||
log.info("Received NER DLQ Message from {} {}", dossierId, fileId);
|
||||
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,9 +1,11 @@
|
||||
package com.iqser.red.service.peristence.v1.server.settings;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileUploadSettings;
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileUploadSettings;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties("persistence-service")
|
||||
public class FileManagementServiceSettings {
|
||||
@ -22,4 +24,5 @@ public class FileManagementServiceSettings {
|
||||
private boolean migrateOnly;
|
||||
|
||||
private boolean imageServiceEnabled = true;
|
||||
private boolean nerServiceEnabled = true;
|
||||
}
|
||||
|
||||
@ -1,5 +1,10 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
|
||||
@ -11,10 +16,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
@ -49,7 +50,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
|
||||
.manualRedactions(new ManualRedactions())
|
||||
.messageType(MessageType.FULL_ANALYSE)
|
||||
.messageType(MessageType.ANALYSE)
|
||||
.analysisVersion(100)
|
||||
.fileId(file.getId())
|
||||
.dossierId(dossier.getId())
|
||||
@ -99,7 +100,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
|
||||
.manualRedactions(new ManualRedactions())
|
||||
.messageType(MessageType.FULL_ANALYSE)
|
||||
.messageType(MessageType.ANALYSE)
|
||||
.analysisVersion(100)
|
||||
.fileId(file.getId())
|
||||
.dossierId(dossier.getId())
|
||||
@ -141,7 +142,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
|
||||
.manualRedactions(new ManualRedactions())
|
||||
.messageType(MessageType.FULL_ANALYSE)
|
||||
.messageType(MessageType.ANALYSE)
|
||||
.analysisVersion(100)
|
||||
.fileId(file.getId())
|
||||
.dossierId(dossier.getId())
|
||||
|
||||
@ -25,7 +25,7 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<redaction-service.version>3.72.0</redaction-service.version>
|
||||
<redaction-service.version>3.74.0</redaction-service.version>
|
||||
<search-service.version>2.18.0</search-service.version>
|
||||
<pdftron-redaction-service.version>3.31.0</pdftron-redaction-service.version>
|
||||
<redaction-report-service.version>3.19.0</redaction-report-service.version>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user