From 1f7b72183cd967d80d3c3c8890646a906578b3c8 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 17:48:17 +0300 Subject: [PATCH] HOTFIX: RED-3800 internal admin api --- .../dossier/file/FileType.java | 11 ++- .../internal/AdminInterfaceController.java | 56 ++++++++++++++ .../DictionaryToEntityMigration2.java | 10 ++- .../v1/server/service/FileStatusService.java | 77 +++++++------------ .../service/job/AutomaticAnalysisJob.java | 19 ++--- 5 files changed, 106 insertions(+), 67 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java index def75e2ca..0e1579b21 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java @@ -3,7 +3,16 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d import lombok.Getter; public enum FileType { - UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json"); + + UNTOUCHED(".pdf"), + ORIGIN(".pdf"), + REDACTION_LOG(".json"), + SECTION_GRID(".json"), + TEXT(".json"), + NER_ENTITIES(".json"), + IMAGE_INFO(".json"), + IMPORTED_REDACTIONS(".json"), + TEXT_HIGHLIGHTS(".json"); @Getter private final String extension; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java new file mode 100644 index 000000000..f21df07c9 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.peristence.v1.server.internal; + + +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.peristence.v1.server.service.FileStatusService; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/private/admin") +public class AdminInterfaceController { + + private final FileManagementStorageService fileManagementStorageService; + private final FileStatusService fileStatusService; + private final FileManagementServiceSettings fileManagementServiceSettings; + + + @PostMapping("/reset-file") + public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + + + } + + + @PostMapping("/reset-image-info") + public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + if (fileManagementServiceSettings.isImageServiceEnabled()) { + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); + fileStatusService.addToImageQueue(dossierId, fileId); + } + } + + + @PostMapping("/ocr") + public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileStatusService.updateLastOCRTime(fileId); + fileStatusService.setStatusOcrProcessing(dossierId, fileId); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 25335596a..08c7c59dc 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -75,9 +75,13 @@ public class DictionaryToEntityMigration2 extends Migration { newRedactionLogEntries.add(entry); }); - redactionLog.setRedactionLogEntry(newRedactionLogEntries); - fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); - log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + + if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) { + redactionLog.setRedactionLogEntry(newRedactionLogEntries); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); + log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + } + } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); } catch (NotFoundException e) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 476b4da70..e2330f0c8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,43 +1,21 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; - -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import javax.transaction.Transactional; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; -import com.iqser.red.service.peristence.v1.server.controller.RulesController; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -45,9 +23,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -58,7 +47,6 @@ public class FileStatusService { private final DossierPersistenceService dossierPersistenceService; private final RabbitTemplate rabbitTemplate; private final ObjectMapper objectMapper; - private final RulesController rulesController; private final ManualRedactionProviderService manualRedactionProviderService; private final FileManagementStorageService fileManagementStorageService; private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; @@ -127,7 +115,6 @@ public class FileStatusService { } - public void setStatusProcessing(String fileId) { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING); @@ -199,7 +186,7 @@ public class FileStatusService { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); - if(requiresStructureAnalysis) { + if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); @@ -213,14 +200,16 @@ public class FileStatusService { protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var fileStatus = fileStatusPersistenceService.getStatus(fileId); + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel); - if (fileStatus.isExcluded()) { - log.debug("File {} is excluded", fileStatus.getId()); + if (fileModel.isExcluded()) { + log.debug("File {} is excluded", fileModel.getId()); return; } - boolean reanalyse = isReanalyse(dossier, fileStatus); + boolean reanalyse = fileModel.isReanalysisRequired(); if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); @@ -228,7 +217,7 @@ public class FileStatusService { return; } - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); + MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus()); if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); @@ -239,26 +228,26 @@ public class FileStatusService { var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) .dossierId(dossierId) - .analysisNumber(fileStatus.getNumberOfAnalyses() + 1) + .analysisNumber(fileModel.getNumberOfAnalyses() + 1) .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) .dossierTemplateId(dossier.getDossierTemplateId()) - .lastProcessed(fileStatus.getLastProcessed()) - .fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileStatus.getExcludedPages()) + .lastProcessed(fileModel.getLastProcessed()) + .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) + .excludedPages(fileModel.getExcludedPages()) .build(); log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); - if(messageType.equals(MessageType.REANALYSE)) { + if (messageType.equals(MessageType.REANALYSE)) { setStatusProcessing(fileId); } else { setStatusFullProcessing(fileId); } try { - if(priority){ + if (priority) { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest)); } else { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest)); @@ -269,16 +258,6 @@ public class FileStatusService { } - private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) { - - return !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() - .isAfter(fileStatus.getLastFileAttributeChange())); - } - - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { @@ -413,7 +392,7 @@ public class FileStatusService { } - private void addToImageQueue(String dossierId, String fileId) { + public void addToImageQueue(String dossierId, String fileId) { setStatusImageAnalyzing(fileId); try { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java index 21ffc26ca..eeb92d83f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java @@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.amqp.core.AmqpAdmin; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.annotation.PostConstruct; import java.util.List; @Slf4j @@ -21,17 +19,9 @@ import java.util.List; @RequiredArgsConstructor public class AutomaticAnalysisJob implements Job { - @Value("${persistence-service.automaticAnalysis.pageFactor:500}") - private int pageFactor; - private final FileStatusService fileStatusService; private final AmqpAdmin amqpAdmin; private final FileManagementServiceSettings fileManagementServiceSettings; - - - @PostConstruct - protected void postConstruct() { - log.info("Automatic Analysis pageFactor: {}", pageFactor); - } + private final FileStatusService fileStatusService; @Override @@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job { var allStatusesIterator = allStatuses.iterator(); log.info("Files that require reanalysis: {}", allStatuses.size()); - var worstCaseScenarioQueuedPages = 0; + var queuedFiles = 0; - while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) { + while (queuedFiles < (consumerCount * 5)) { var next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. - worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages(); if (next.isFullAnalysisRequired()) { log.info("Queued file: {} for automatic full analysis! ", next.getFilename()); @@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job { log.info("Queued file: {} for automatic reanalysis! ", next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false); } + + queuedFiles++; } }