From 2023ac13b97f8ff10ce5c8d96c7fcad3f2aa507c Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Tue, 10 Jan 2023 12:40:03 +0100 Subject: [PATCH 1/3] RED-5892: Corrected next processing step calculation. Changed the calculation to always re-produce NER-entities if they are missing, instead of only doing that for full re-processing. --- .../v1/server/service/FileStatusService.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 34fb1b22e..a0a2b49d3 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -577,11 +577,6 @@ public class FileStatusService { return; } - var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - - boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; - if (settings.isFigureDetectionEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.FIGURE)) { log.debug("Add file: {} from dossier {} to Figure Detection queue", fileId, dossierId); addToFigureDetectionRequestQueue(dossierId, fileId); @@ -600,20 +595,32 @@ public class FileStatusService { return; } + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + if (settings.isOcrByDefault() && fileModel.getOcrEndTime() == null) { log.debug("Add file: {} from dossier {} to OCR queue", fileId, dossierId); setStatusOcrQueued(dossierId, fileId); return; } - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); + MessageType messageType = null; - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { + messageType = MessageType.STRUCTURE_ANALYSE; + } + + if (messageType == null && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; } + if (messageType == null) { + boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; + messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); + } + var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) .dossierId(dossierId) @@ -670,18 +677,18 @@ public class FileStatusService { private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) { - if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { - return MessageType.STRUCTURE_ANALYSE; - } if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { return MessageType.ANALYSE; } + if (reanalyse) { if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) { return MessageType.ANALYSE; + } else { + return MessageType.REANALYSE; } - return MessageType.REANALYSE; } + return MessageType.ANALYSE; } From 5e7bd282cfbea63e478649b559b1a6c0a156c73b Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Tue, 10 Jan 2023 14:03:48 +0100 Subject: [PATCH 2/3] RED-5892: Removed unused method params --- .../peristence/v1/server/service/FileStatusService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index a0a2b49d3..5f3c1ff2e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -618,7 +618,7 @@ public class FileStatusService { if (messageType == null) { boolean reanalyse = fileModel.isReanalysisRequired() || manualRedactionReanalyse; - messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); + messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel); } var analyseRequest = AnalyzeRequest.builder() @@ -675,7 +675,7 @@ public class FileStatusService { } - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) { + private MessageType calculateMessageType(boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) { if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { return MessageType.ANALYSE; From af9f68d498a18f6a5069b9674016db347f2a6189 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Tue, 10 Jan 2023 17:49:45 +0100 Subject: [PATCH 3/3] RED-5892: Consolidated attribute changes and state changes. Moved the change to the full-reprocess-status after a exclusion or auto-analysis flag toggle into the FileStatusService, so that the changes can be run in a transaction, which will prevent other threads from seeing incomplete changes. --- .../service/ExcludeFromAnalysisService.java | 17 ++--------------- .../v1/server/service/FileStatusService.java | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java index 083b2351e..49b84317a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java @@ -23,26 +23,13 @@ public class ExcludeFromAnalysisService { } // toggle status - fileStatusService.toggleExclusion(fileId, excluded); - - if (!excluded) { - // if file has been re-enabled - process it - fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true); - } - + fileStatusService.toggleExclusion(dossierId, fileId, excluded); } public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) { - // toggle status - fileStatusService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis); - - if (!excludedFromAutomaticAnalysis) { - // if file has been re-enabled - process it - fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true); - } - + fileStatusService.toggleAutomaticAnalysis(dossierId, fileId, excludedFromAutomaticAnalysis); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 5f3c1ff2e..f6558fcf2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -350,15 +350,25 @@ public class FileStatusService { } - public void toggleExclusion(String fileId, boolean excluded) { + @Transactional + public void toggleExclusion(String dossierId, String fileId, boolean excluded) { fileStatusPersistenceService.toggleExclusion(fileId, excluded); + + if (!excluded) { + setStatusFullReprocess(dossierId, fileId, false, true); + } } - public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) { + @Transactional + public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) { fileStatusPersistenceService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis); + + if (!excludedFromAutomaticAnalysis) { + setStatusFullReprocess(dossierId, fileId, false, true); + } }