diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 93b8ecf16..25335596a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -1,10 +1,5 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.migration.Migration; @@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; @Slf4j @Setter @@ -56,8 +53,11 @@ public class DictionaryToEntityMigration2 extends Migration { files.forEach(file -> { log.info("Start migration of file {}", file.getId()); if (file.getHardDeletedTime() == null) { - var newRedactionLogEntries = new ArrayList(); + try { + + var newRedactionLogEntries = new ArrayList(); + var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); redactionLog.getRedactionLogEntry().forEach(entry -> { @@ -69,7 +69,7 @@ public class DictionaryToEntityMigration2 extends Migration { if (entry.getType().startsWith("recommendation_")) { entry.setType(entry.getType().substring(15)); entry.setRecommendation(true); - log.info("removed _recommendation"); + log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId()); } newRedactionLogEntries.add(entry); @@ -77,12 +77,11 @@ public class DictionaryToEntityMigration2 extends Migration { redactionLog.setRedactionLogEntry(newRedactionLogEntries); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); - log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); } catch (NotFoundException e) { - log.info("redactionLog {} does not exsist", file.getId()); - return; + log.info("RedactionLog does not exist for file with id {}.", file.getId()); } } }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index 61434eb7c..cca7d5612 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -48,9 +48,13 @@ public class MigrateHighlights3 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null) { - var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); - var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); - fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { + var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); + var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); + fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + } else { + log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); + } } }); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java index 62bdeff36..2e423ddf4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java @@ -59,6 +59,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration { } }); } + + log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(),annotationIdsToRemove); + if (!annotationIdsToRemove.isEmpty()) { annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id)); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java index 81891338f..45e9d5537 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java @@ -65,6 +65,7 @@ public class TypeToEntityMigration5 extends Migration { .filter(t -> t.getType().equals("false_positive")) .findFirst(); if (falsePositive.isEmpty()) { + log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName()); return; } typeIdsToDelete.add(falsePositive.get().getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java index 8f3f7b322..f37c6f793 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java @@ -101,22 +101,8 @@ public class FileManagementStorageService { } } - - public boolean imageInfoExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); - } - - - public boolean textExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); - } - - - public boolean nerEntitiesExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); + public boolean objectExists(String dossierId, String fileId, FileType origin) { + return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin)); } @@ -131,4 +117,5 @@ public class FileManagementStorageService { storageService.deleteObject(storageId); } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 532d0fefd..1734e3d31 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.bind.annotation.RestController; @@ -34,7 +35,7 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { case STRUCTURE_ANALYSE: - if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { fileStatusService.addToNerQueue(dossierId, fileId); } else { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index d6a8a625d..476b4da70 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -222,7 +222,7 @@ public class FileStatusService { boolean reanalyse = isReanalyse(dossier, fileStatus); - if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) { + if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); addToImageQueue(dossierId, fileId); return; @@ -230,7 +230,7 @@ public class FileStatusService { MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; @@ -282,7 +282,7 @@ public class FileStatusService { private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { - if (!fileManagementStorageService.textExists(dossierId, fileId)) { + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; } if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {