From 634c1c98dc3d93f2241b89ae425818fcd30f22a1 Mon Sep 17 00:00:00 2001 From: Christoph Schabert Date: Fri, 29 Apr 2022 10:17:12 +0200 Subject: [PATCH 01/10] hotfix: migration to skip missing redaction logs (e.g. for deleted files w/o redaction log) --- .../DictionaryToEntityMigration2.java | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 1696b2998..93b8ecf16 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; @@ -48,7 +49,6 @@ public class DictionaryToEntityMigration2 extends Migration { @Override protected void migrate() { - var dossiers = dossierPersistenceService.findAllDossiers(); dossiers.forEach(dossier -> { var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); @@ -57,32 +57,33 @@ public class DictionaryToEntityMigration2 extends Migration { log.info("Start migration of file {}", file.getId()); if (file.getHardDeletedTime() == null) { var newRedactionLogEntries = new ArrayList(); - var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); - - redactionLog.getRedactionLogEntry().forEach(entry -> { - - if (entry.getType().equals("false_positive")){ - log.info("skipping false_positive for dossier {} and file {}", dossier.getId(), file.getId()); - return; - } - - if (entry.getType().startsWith("recommendation_")) { - entry.setType(entry.getType().substring(15)); - entry.setRecommendation(true); - log.info("removed _recommendation"); - } - - newRedactionLogEntries.add(entry); - }); - - redactionLog.setRedactionLogEntry(newRedactionLogEntries); try { + var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); + redactionLog.getRedactionLogEntry().forEach(entry -> { + + if (entry.getType().equals("false_positive")) { + log.info("skipping false_positive for dossier {} and file {}", dossier.getId(), file.getId()); + return; + } + + if (entry.getType().startsWith("recommendation_")) { + entry.setType(entry.getType().substring(15)); + entry.setRecommendation(true); + log.info("removed _recommendation"); + } + + newRedactionLogEntries.add(entry); + }); + + redactionLog.setRedactionLogEntry(newRedactionLogEntries); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId()); } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); + } catch (NotFoundException e) { + log.info("redactionLog {} does not exsist", file.getId()); + return; } - } }); }); From 874c94d7026544fa7017a6ab937612e4e6c418d9 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 13:29:17 +0300 Subject: [PATCH 02/10] RED-3800 Migration cleanup --- .../DictionaryToEntityMigration2.java | 23 +++++++++---------- .../migrations/MigrateHighlights3.java | 10 +++++--- .../RemoveFalsePositiveManualRedactions6.java | 3 +++ .../migrations/TypeToEntityMigration5.java | 1 + .../service/FileManagementStorageService.java | 19 +++------------ .../FileStatusProcessingUpdateService.java | 3 ++- .../v1/server/service/FileStatusService.java | 6 ++--- 7 files changed, 30 insertions(+), 35 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 93b8ecf16..25335596a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -1,10 +1,5 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.migration.Migration; @@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; @Slf4j @Setter @@ -56,8 +53,11 @@ public class DictionaryToEntityMigration2 extends Migration { files.forEach(file -> { log.info("Start migration of file {}", file.getId()); if (file.getHardDeletedTime() == null) { - var newRedactionLogEntries = new ArrayList(); + try { + + var newRedactionLogEntries = new ArrayList(); + var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); redactionLog.getRedactionLogEntry().forEach(entry -> { @@ -69,7 +69,7 @@ public class DictionaryToEntityMigration2 extends Migration { if (entry.getType().startsWith("recommendation_")) { entry.setType(entry.getType().substring(15)); entry.setRecommendation(true); - log.info("removed _recommendation"); + log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId()); } newRedactionLogEntries.add(entry); @@ -77,12 +77,11 @@ public class DictionaryToEntityMigration2 extends Migration { redactionLog.setRedactionLogEntry(newRedactionLogEntries); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); - log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); } catch (NotFoundException e) { - log.info("redactionLog {} does not exsist", file.getId()); - return; + log.info("RedactionLog does not exist for file with id {}.", file.getId()); } } }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index 61434eb7c..cca7d5612 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -48,9 +48,13 @@ public class MigrateHighlights3 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null) { - var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); - var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); - fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { + var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); + var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); + fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + } else { + log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); + } } }); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java index 62bdeff36..2e423ddf4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java @@ -59,6 +59,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration { } }); } + + log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(),annotationIdsToRemove); + if (!annotationIdsToRemove.isEmpty()) { annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id)); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java index 81891338f..45e9d5537 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java @@ -65,6 +65,7 @@ public class TypeToEntityMigration5 extends Migration { .filter(t -> t.getType().equals("false_positive")) .findFirst(); if (falsePositive.isEmpty()) { + log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName()); return; } typeIdsToDelete.add(falsePositive.get().getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java index 8f3f7b322..f37c6f793 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java @@ -101,22 +101,8 @@ public class FileManagementStorageService { } } - - public boolean imageInfoExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); - } - - - public boolean textExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); - } - - - public boolean nerEntitiesExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); + public boolean objectExists(String dossierId, String fileId, FileType origin) { + return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin)); } @@ -131,4 +117,5 @@ public class FileManagementStorageService { storageService.deleteObject(storageId); } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 532d0fefd..1734e3d31 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.bind.annotation.RestController; @@ -34,7 +35,7 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { case STRUCTURE_ANALYSE: - if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { fileStatusService.addToNerQueue(dossierId, fileId); } else { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index d6a8a625d..476b4da70 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -222,7 +222,7 @@ public class FileStatusService { boolean reanalyse = isReanalyse(dossier, fileStatus); - if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) { + if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); addToImageQueue(dossierId, fileId); return; @@ -230,7 +230,7 @@ public class FileStatusService { MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; @@ -282,7 +282,7 @@ public class FileStatusService { private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { - if (!fileManagementStorageService.textExists(dossierId, fileId)) { + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; } if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { From b82d20473d08e56a1a220496e41a11a36c4b389d Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 14:02:23 +0300 Subject: [PATCH 03/10] NPE fix RED-3800 --- .../migrations/MigrateHighlights3.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index cca7d5612..828117877 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -32,7 +32,6 @@ public class MigrateHighlights3 extends Migration { @Autowired private FileManagementStorageService fileManagementStorageService; - public MigrateHighlights3() { super(NAME, VERSION); @@ -48,12 +47,16 @@ public class MigrateHighlights3 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null) { - if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { - var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); - var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); - fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); - } else { - log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); + try { + if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { + var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); + var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); + fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + } else { + log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); + } + } catch (Exception e) { + log.warn("Failed to extract text highlights for document: {}", file.getId()); } } }); From 5c57634d9c8b3e7cfc8c8a69e5ba46feaa406b71 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 16:59:06 +0300 Subject: [PATCH 04/10] migration manual run ctrl RED-3800 --- .../v1/server/migration/Migration.java | 11 ++++---- .../server/migration/MigrationController.java | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java index 90ec3e2df..540bfbbf0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java @@ -1,13 +1,11 @@ package com.iqser.red.service.peristence.v1.server.migration; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; - import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; @Data @Slf4j @@ -21,11 +19,14 @@ public abstract class Migration { @Setter private MigrationPersistenceService migrationPersistenceService; - public void run() { + run(false); + } + + public void run(boolean force) { var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion(); - if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) { + if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion) || force) { log.info("Starting migration with name {} and version {}", name, version); migrate(); migrationPersistenceService.insertMigration(name, version); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java new file mode 100644 index 000000000..caef196df --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.peristence.v1.server.migration; + + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@Data +@RestController +@RequiredArgsConstructor +@RequestMapping("/private/migration") +public class MigrationController { + + private final List migrations; + + + @PostMapping + public void run(@RequestParam("migrationName") String migrationName, @RequestParam("force") boolean force) { + migrations.stream().filter(m -> m.getName().equalsIgnoreCase(migrationName) || m.getClass().getSimpleName().equalsIgnoreCase(migrationName)) + .forEach(m -> m.run(force)); + } + +} From 1f7b72183cd967d80d3c3c8890646a906578b3c8 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 17:48:17 +0300 Subject: [PATCH 05/10] HOTFIX: RED-3800 internal admin api --- .../dossier/file/FileType.java | 11 ++- .../internal/AdminInterfaceController.java | 56 ++++++++++++++ .../DictionaryToEntityMigration2.java | 10 ++- .../v1/server/service/FileStatusService.java | 77 +++++++------------ .../service/job/AutomaticAnalysisJob.java | 19 ++--- 5 files changed, 106 insertions(+), 67 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java index def75e2ca..0e1579b21 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java @@ -3,7 +3,16 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d import lombok.Getter; public enum FileType { - UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json"); + + UNTOUCHED(".pdf"), + ORIGIN(".pdf"), + REDACTION_LOG(".json"), + SECTION_GRID(".json"), + TEXT(".json"), + NER_ENTITIES(".json"), + IMAGE_INFO(".json"), + IMPORTED_REDACTIONS(".json"), + TEXT_HIGHLIGHTS(".json"); @Getter private final String extension; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java new file mode 100644 index 000000000..f21df07c9 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.peristence.v1.server.internal; + + +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.peristence.v1.server.service.FileStatusService; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/private/admin") +public class AdminInterfaceController { + + private final FileManagementStorageService fileManagementStorageService; + private final FileStatusService fileStatusService; + private final FileManagementServiceSettings fileManagementServiceSettings; + + + @PostMapping("/reset-file") + public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + + + } + + + @PostMapping("/reset-image-info") + public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + if (fileManagementServiceSettings.isImageServiceEnabled()) { + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); + fileStatusService.addToImageQueue(dossierId, fileId); + } + } + + + @PostMapping("/ocr") + public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileStatusService.updateLastOCRTime(fileId); + fileStatusService.setStatusOcrProcessing(dossierId, fileId); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 25335596a..08c7c59dc 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -75,9 +75,13 @@ public class DictionaryToEntityMigration2 extends Migration { newRedactionLogEntries.add(entry); }); - redactionLog.setRedactionLogEntry(newRedactionLogEntries); - fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); - log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + + if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) { + redactionLog.setRedactionLogEntry(newRedactionLogEntries); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); + log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + } + } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); } catch (NotFoundException e) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 476b4da70..e2330f0c8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,43 +1,21 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; - -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import javax.transaction.Transactional; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; -import com.iqser.red.service.peristence.v1.server.controller.RulesController; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -45,9 +23,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -58,7 +47,6 @@ public class FileStatusService { private final DossierPersistenceService dossierPersistenceService; private final RabbitTemplate rabbitTemplate; private final ObjectMapper objectMapper; - private final RulesController rulesController; private final ManualRedactionProviderService manualRedactionProviderService; private final FileManagementStorageService fileManagementStorageService; private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; @@ -127,7 +115,6 @@ public class FileStatusService { } - public void setStatusProcessing(String fileId) { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING); @@ -199,7 +186,7 @@ public class FileStatusService { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); - if(requiresStructureAnalysis) { + if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); @@ -213,14 +200,16 @@ public class FileStatusService { protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var fileStatus = fileStatusPersistenceService.getStatus(fileId); + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel); - if (fileStatus.isExcluded()) { - log.debug("File {} is excluded", fileStatus.getId()); + if (fileModel.isExcluded()) { + log.debug("File {} is excluded", fileModel.getId()); return; } - boolean reanalyse = isReanalyse(dossier, fileStatus); + boolean reanalyse = fileModel.isReanalysisRequired(); if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); @@ -228,7 +217,7 @@ public class FileStatusService { return; } - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); + MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus()); if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); @@ -239,26 +228,26 @@ public class FileStatusService { var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) .dossierId(dossierId) - .analysisNumber(fileStatus.getNumberOfAnalyses() + 1) + .analysisNumber(fileModel.getNumberOfAnalyses() + 1) .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) .dossierTemplateId(dossier.getDossierTemplateId()) - .lastProcessed(fileStatus.getLastProcessed()) - .fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileStatus.getExcludedPages()) + .lastProcessed(fileModel.getLastProcessed()) + .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) + .excludedPages(fileModel.getExcludedPages()) .build(); log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); - if(messageType.equals(MessageType.REANALYSE)) { + if (messageType.equals(MessageType.REANALYSE)) { setStatusProcessing(fileId); } else { setStatusFullProcessing(fileId); } try { - if(priority){ + if (priority) { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest)); } else { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest)); @@ -269,16 +258,6 @@ public class FileStatusService { } - private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) { - - return !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() - .isAfter(fileStatus.getLastFileAttributeChange())); - } - - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { @@ -413,7 +392,7 @@ public class FileStatusService { } - private void addToImageQueue(String dossierId, String fileId) { + public void addToImageQueue(String dossierId, String fileId) { setStatusImageAnalyzing(fileId); try { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java index 21ffc26ca..eeb92d83f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java @@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.amqp.core.AmqpAdmin; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.annotation.PostConstruct; import java.util.List; @Slf4j @@ -21,17 +19,9 @@ import java.util.List; @RequiredArgsConstructor public class AutomaticAnalysisJob implements Job { - @Value("${persistence-service.automaticAnalysis.pageFactor:500}") - private int pageFactor; - private final FileStatusService fileStatusService; private final AmqpAdmin amqpAdmin; private final FileManagementServiceSettings fileManagementServiceSettings; - - - @PostConstruct - protected void postConstruct() { - log.info("Automatic Analysis pageFactor: {}", pageFactor); - } + private final FileStatusService fileStatusService; @Override @@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job { var allStatusesIterator = allStatuses.iterator(); log.info("Files that require reanalysis: {}", allStatuses.size()); - var worstCaseScenarioQueuedPages = 0; + var queuedFiles = 0; - while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) { + while (queuedFiles < (consumerCount * 5)) { var next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. - worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages(); if (next.isFullAnalysisRequired()) { log.info("Queued file: {} for automatic full analysis! ", next.getFilename()); @@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job { log.info("Queued file: {} for automatic reanalysis! ", next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false); } + + queuedFiles++; } } From d96c9cae41eca4c22aa7c3c7868870f1bfbeb64f Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 18:38:28 +0300 Subject: [PATCH 06/10] HOTFIX: RED-3800 ocr all files that have been ocrd --- .../entity/dossier/DossierEntity.java | 3 ++ .../MigrationPersistenceService.java | 9 ++--- .../internal/AdminInterfaceController.java | 40 +++++++++++++++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java index 0defeeea8..f74a1767b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java @@ -113,4 +113,7 @@ public class DossierEntity { @Column(updatable = false, insertable = false, name = "dossier_status_id") private String dossierStatusId; + public boolean isDeleted() { + return softDeletedTime != null || hardDeletedTime != null; + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java index 87e2b1956..0870ea4da 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java @@ -29,20 +29,17 @@ public class MigrationPersistenceService { var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion); if (migrations == null || migrations.isEmpty()) { - if(version > lastProcessedVersion){ - return false; - } - return true; + return version <= lastProcessedVersion; } - return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent(); + return migrations.stream().anyMatch(m -> m.getVersion() == version); } public Long getLatestProcessedVersion() { var migrations = migrationRepository.findAll(); - if (migrations == null || migrations.isEmpty()) { + if (migrations.isEmpty()) { return null; } return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java index f21df07c9..39d99ae9b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -1,16 +1,22 @@ package com.iqser.red.service.peristence.v1.server.internal; +import com.iqser.red.service.peristence.v1.server.service.DossierService; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.stream.Collectors; + +@Slf4j @RestController @RequiredArgsConstructor @RequestMapping("/private/admin") @@ -20,6 +26,8 @@ public class AdminInterfaceController { private final FileStatusService fileStatusService; private final FileManagementServiceSettings fileManagementServiceSettings; + private final DossierService dossierService; + @PostMapping("/reset-file") public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { @@ -53,4 +61,36 @@ public class AdminInterfaceController { } + + @PostMapping("/ocr-all-files-that-have-already-been-ocrd") + public void forceOCR(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) { + + var dossiers = dossierService.getAllDossiers(); + for (var dossier : dossiers) { + if (!dossier.isDeleted()) { + var files = fileStatusService.getActiveFiles(dossier.getId()); + var filesThatRequireOCR = files.stream() + .filter(f -> !f.isExcluded()) + .filter(f -> !f.isExcludedFromAutomaticAnalysis()) + .filter(f -> !f.isSoftOrHardDeleted()) + .filter(f -> f.getLastOCRTime() != null) + .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList()); + + + for (var file : filesThatRequireOCR) { + log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + + if (!dryRun) { + fileStatusService.updateLastOCRTime(file.getId()); + fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId()); + } + + } + + + } + } + } + } From 6d5927f667c0322a8acaabd05bbc3b022d420cf7 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 18:43:32 +0300 Subject: [PATCH 07/10] HOTFIX RED-3800 endpoint to reset text for all files that are not approved/deleted --- .../internal/AdminInterfaceController.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java index 39d99ae9b..721a76cc4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -65,6 +65,10 @@ public class AdminInterfaceController { @PostMapping("/ocr-all-files-that-have-already-been-ocrd") public void forceOCR(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) { + if (dryRun) { + log.info("Dry run!"); + } + var dossiers = dossierService.getAllDossiers(); for (var dossier : dossiers) { if (!dossier.isDeleted()) { @@ -93,4 +97,47 @@ public class AdminInterfaceController { } } + + @PostMapping("/reset-text-for-all-files") + public void resetText(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) { + + if (dryRun) { + log.info("Dry run!"); + } + + var dossiers = dossierService.getAllDossiers(); + for (var dossier : dossiers) { + if (!dossier.isDeleted()) { + var files = fileStatusService.getActiveFiles(dossier.getId()); + var filesThatRequireTextReset = files.stream() + .filter(f -> !f.isExcluded()) + .filter(f -> !f.isExcludedFromAutomaticAnalysis()) + .filter(f -> !f.isSoftOrHardDeleted()) + .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList()); + + + for (var file : filesThatRequireTextReset) { + log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + + if (!dryRun) { + + var dossierId = file.getDossierId(); + var fileId = file.getId(); + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + } + + } + + + } + } + } + } From 342b2dddcd7699aa6e23e7c8dbd8f2d5c8a0e7be Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 19:51:35 +0300 Subject: [PATCH 08/10] Hotfix RED-3800 reanslysis required status updates --- .../controller/ReanalysisController.java | 4 +- .../v1/server/service/FileStatusService.java | 12 +-- .../service/ManualRedactionService.java | 27 +++--- .../ReanalysisRequiredStatusService.java | 19 +++- .../tests/ManualRedactionTest.java | 94 ++++++++++++++----- 5 files changed, 109 insertions(+), 47 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index 6267635f6..5ef803f0d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -230,11 +230,11 @@ public class ReanalysisController implements ReanalysisResource { if (force) { filesToReanalyse.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } else { filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index e2330f0c8..f081914cf 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -184,8 +184,6 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); - if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); @@ -201,14 +199,16 @@ public class FileStatusService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); - var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel); - if (fileModel.isExcluded()) { - log.debug("File {} is excluded", fileModel.getId()); + if (fileEntity.isExcluded()) { + log.debug("File {} is excluded", fileEntity.getId()); return; } + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + + boolean reanalyse = fileModel.isReanalysisRequired(); if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index fd1aaa0e2..ff9716244 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -127,23 +127,26 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); + }else { + + var manualTextRedactions = convert(response.stream() + .map(r -> getAddRedaction(fileId, r.getAnnotationId())) + .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); + + if (!manualTextRedactions.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); + } + } - var manualTextRedactions = convert(response.stream() - .map(r -> getAddRedaction(fileId, r.getAnnotationId())) - .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) - .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); - - if (!manualTextRedactions.isEmpty()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - return response; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index c86b3bcca..00f81533a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService { public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { - return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next(); + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next(); + } + + public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) { + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next(); } public List enhanceFileStatusWithAnalysisRequirements(List fileModels) { + return enhanceFileStatusWithAnalysisRequirements(fileModels, false); + } + + public List enhanceFileStatusWithAnalysisRequirements(List fileModels, boolean ignoreProcessingStates) { Map> dossierTemplateVersionMap = new HashMap<>(); Map dossierVersionMap = new HashMap<>(); Map dossierMap = new HashMap<>(); fileModels.forEach(entry -> { - var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap); + var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap); entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired()); entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); }); @@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService { } private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus, + boolean ignoreProcessingStates, Map> dossierTemplateVersionMap, Map dossierVersionMap, Map dossierMap) { - if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) { return new AnalysisRequiredResult(false, true); } - if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) + || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus()) + || ignoreProcessingStates) { switch (fileStatus.getWorkflowStatus()) { case NEW: diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 5c32f5f11..f2fb154a4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1,38 +1,40 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.redaction.v1.model.ManualChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import feign.FeignException; -import org.assertj.core.util.Lists; -import org.junit.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.redaction.v1.model.ManualChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - +import feign.FeignException; import lombok.SneakyThrows; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Autowired private FileManagementStorageService fileManagementStorageService; + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + @Autowired private ObjectMapper objectMapper; @@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } + + @Test + @SneakyThrows + public void testAddToDictionaryRequiresReanalysis() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII"); + + + + // assume file is already proccessed once, test that add to dict triggers reanalysis + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8)); + fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED); + + var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder() + .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) + .section("section test") + .addToDictionary(true) + .addToDossierDictionary(false) + .status(AnnotationStatus.APPROVED) + .typeId(type.getId()) + .user("user") + .reason("1") + .value("test") + .legalBasis("1") + .rectangle(true) + .textAfter("Text After") + .textBefore("Text Before") + .sourceId("SourceId") + .build())).iterator().next(); + + + var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + } + @Test @SneakyThrows public void testManualRedaction() { @@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .annotationStatus(AnnotationStatus.REQUESTED).build()); var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId()); assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED); - assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty()); + assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty()); assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true); manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(), @@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertThat(manualRedactions.getComments()).isNotEmpty(); assertThat(manualRedactions.getResizeRedactions()).isNotEmpty(); - List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); + List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds); // manualRedactions.getForceRedactions() // .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId()))); From 8ac33bc0fe1b846a9f4e1e5ffc025daaee726881 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 29 Apr 2022 20:17:50 +0300 Subject: [PATCH 09/10] Hotfix RED-3800 purge queue --- .../v1/server/internal/AdminInterfaceController.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java index 721a76cc4..e7dea6f76 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -9,6 +9,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.core.AmqpAdmin; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -25,8 +26,9 @@ public class AdminInterfaceController { private final FileManagementStorageService fileManagementStorageService; private final FileStatusService fileStatusService; private final FileManagementServiceSettings fileManagementServiceSettings; - private final DossierService dossierService; + private final AmqpAdmin amqpAdmin; + @PostMapping("/reset-file") @@ -140,4 +142,10 @@ public class AdminInterfaceController { } } + @PostMapping("/flush-queue") + public void resetText(@RequestParam() String queueName ) { + amqpAdmin.purgeQueue(queueName); + } + + } From cd41c3e164464313ed241403e266f2a3ee4657ec Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Tue, 3 May 2022 17:32:55 +0300 Subject: [PATCH 10/10] RED-3800 code cleanup --- .../v1/api/resources/DossierTemplateStatsResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java index fd1cf97ac..81642b6ac 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java @@ -19,10 +19,10 @@ public interface DossierTemplateStatsResource { String DOSSIER_TEMPLATE_ID = "dossierTemplateId"; String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}"; - @PostMapping(value = REST_PATH+DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @PostMapping(value = REST_PATH + DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) List getDossierTemplateDictionaryStats(@RequestBody Set dossierTemplateIds); - @PostMapping(value = REST_PATH+DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE ) + @PostMapping(value = REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) DossierTemplateStats getDossierTemplateStats(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId); @PostMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)