HOTFIX: RED-3800 internal admin api
This commit is contained in:
Timo Bejan
parent
5c57634d9c
commit
1f7b72183c
@ -3,7 +3,16 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d
|
||||
import lombok.Getter;
|
||||
|
||||
public enum FileType {
|
||||
UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json");
|
||||
|
||||
UNTOUCHED(".pdf"),
|
||||
ORIGIN(".pdf"),
|
||||
REDACTION_LOG(".json"),
|
||||
SECTION_GRID(".json"),
|
||||
TEXT(".json"),
|
||||
NER_ENTITIES(".json"),
|
||||
IMAGE_INFO(".json"),
|
||||
IMPORTED_REDACTIONS(".json"),
|
||||
TEXT_HIGHLIGHTS(".json");
|
||||
|
||||
@Getter
|
||||
private final String extension;
|
||||
|
||||
@ -0,0 +1,56 @@
|
||||
package com.iqser.red.service.peristence.v1.server.internal;
|
||||
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/private/admin")
|
||||
public class AdminInterfaceController {
|
||||
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final FileStatusService fileStatusService;
|
||||
private final FileManagementServiceSettings fileManagementServiceSettings;
|
||||
|
||||
|
||||
@PostMapping("/reset-file")
|
||||
public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
|
||||
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
|
||||
|
||||
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("/reset-image-info")
|
||||
public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
|
||||
|
||||
if (fileManagementServiceSettings.isImageServiceEnabled()) {
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
|
||||
fileStatusService.addToImageQueue(dossierId, fileId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("/ocr")
|
||||
public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
|
||||
|
||||
fileStatusService.updateLastOCRTime(fileId);
|
||||
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -75,9 +75,13 @@ public class DictionaryToEntityMigration2 extends Migration {
|
||||
newRedactionLogEntries.add(entry);
|
||||
});
|
||||
|
||||
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
|
||||
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
|
||||
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
|
||||
|
||||
if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
|
||||
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
|
||||
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
|
||||
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
|
||||
}
|
||||
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new RuntimeException("Migration failed");
|
||||
} catch (NotFoundException e) {
|
||||
|
||||
@ -1,43 +1,21 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.peristence.v1.server.controller.RulesController;
|
||||
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
|
||||
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
@ -45,9 +23,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.MessageType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -58,7 +47,6 @@ public class FileStatusService {
|
||||
private final DossierPersistenceService dossierPersistenceService;
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RulesController rulesController;
|
||||
private final ManualRedactionProviderService manualRedactionProviderService;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
|
||||
@ -127,7 +115,6 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setStatusProcessing(String fileId) {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
|
||||
@ -199,7 +186,7 @@ public class FileStatusService {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
|
||||
|
||||
if(requiresStructureAnalysis) {
|
||||
if (requiresStructureAnalysis) {
|
||||
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
|
||||
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
|
||||
@ -213,14 +200,16 @@ public class FileStatusService {
|
||||
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
|
||||
|
||||
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
|
||||
var fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
|
||||
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
|
||||
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel);
|
||||
|
||||
if (fileStatus.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileStatus.getId());
|
||||
if (fileModel.isExcluded()) {
|
||||
log.debug("File {} is excluded", fileModel.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
boolean reanalyse = isReanalyse(dossier, fileStatus);
|
||||
boolean reanalyse = fileModel.isReanalysisRequired();
|
||||
|
||||
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
|
||||
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
|
||||
@ -228,7 +217,7 @@ public class FileStatusService {
|
||||
return;
|
||||
}
|
||||
|
||||
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
|
||||
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus());
|
||||
|
||||
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
|
||||
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
|
||||
@ -239,26 +228,26 @@ public class FileStatusService {
|
||||
var analyseRequest = AnalyzeRequest.builder()
|
||||
.messageType(messageType)
|
||||
.dossierId(dossierId)
|
||||
.analysisNumber(fileStatus.getNumberOfAnalyses() + 1)
|
||||
.analysisNumber(fileModel.getNumberOfAnalyses() + 1)
|
||||
.sectionsToReanalyse(sectionsToReanalyse)
|
||||
.fileId(fileId)
|
||||
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
|
||||
.dossierTemplateId(dossier.getDossierTemplateId())
|
||||
.lastProcessed(fileStatus.getLastProcessed())
|
||||
.fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId()))
|
||||
.excludedPages(fileStatus.getExcludedPages())
|
||||
.lastProcessed(fileModel.getLastProcessed())
|
||||
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
|
||||
.excludedPages(fileModel.getExcludedPages())
|
||||
.build();
|
||||
|
||||
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
|
||||
|
||||
if(messageType.equals(MessageType.REANALYSE)) {
|
||||
if (messageType.equals(MessageType.REANALYSE)) {
|
||||
setStatusProcessing(fileId);
|
||||
} else {
|
||||
setStatusFullProcessing(fileId);
|
||||
}
|
||||
|
||||
try {
|
||||
if(priority){
|
||||
if (priority) {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest));
|
||||
} else {
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest));
|
||||
@ -269,16 +258,6 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) {
|
||||
|
||||
return !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
|
||||
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
|
||||
.isAfter(fileStatus.getLastFileAttributeChange()));
|
||||
}
|
||||
|
||||
|
||||
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
|
||||
ProcessingStatus processingStatus) {
|
||||
|
||||
@ -413,7 +392,7 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
private void addToImageQueue(String dossierId, String fileId) {
|
||||
public void addToImageQueue(String dossierId, String fileId) {
|
||||
|
||||
setStatusImageAnalyzing(fileId);
|
||||
try {
|
||||
|
||||
@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.springframework.amqp.core.AmqpAdmin;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@ -21,17 +19,9 @@ import java.util.List;
|
||||
@RequiredArgsConstructor
|
||||
public class AutomaticAnalysisJob implements Job {
|
||||
|
||||
@Value("${persistence-service.automaticAnalysis.pageFactor:500}")
|
||||
private int pageFactor;
|
||||
private final FileStatusService fileStatusService;
|
||||
private final AmqpAdmin amqpAdmin;
|
||||
private final FileManagementServiceSettings fileManagementServiceSettings;
|
||||
|
||||
|
||||
@PostConstruct
|
||||
protected void postConstruct() {
|
||||
log.info("Automatic Analysis pageFactor: {}", pageFactor);
|
||||
}
|
||||
private final FileStatusService fileStatusService;
|
||||
|
||||
|
||||
@Override
|
||||
@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job {
|
||||
var allStatusesIterator = allStatuses.iterator();
|
||||
log.info("Files that require reanalysis: {}", allStatuses.size());
|
||||
|
||||
var worstCaseScenarioQueuedPages = 0;
|
||||
var queuedFiles = 0;
|
||||
|
||||
while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) {
|
||||
while (queuedFiles < (consumerCount * 5)) {
|
||||
var next = allStatusesIterator.next();
|
||||
// in case the file doesn't have numberOfPages set, we assume an average.
|
||||
worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
|
||||
|
||||
if (next.isFullAnalysisRequired()) {
|
||||
log.info("Queued file: {} for automatic full analysis! ", next.getFilename());
|
||||
@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job {
|
||||
log.info("Queued file: {} for automatic reanalysis! ", next.getFilename());
|
||||
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false);
|
||||
}
|
||||
|
||||
queuedFiles++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user