From 87f841d9dd679d7e693c3e9c787c0913a9dc049d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Mon, 11 Dec 2023 15:46:32 +0100 Subject: [PATCH] RED-7834: fix migration issues --- .../controller/MigrationStatusController.java | 58 ++++++++- .../build.gradle.kts | 2 +- .../resource/MigrationStatusResource.java | 23 ++++ .../build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../configuration/MessagingConfiguration.java | 3 +- .../migration/SaasMigrationService.java | 114 +++++++++++++----- .../service/EntityLogMergeService.java | 8 +- .../processor/service/FileStatusService.java | 5 + .../FileStatusPersistenceService.java | 6 + .../LayoutParsingFinishedMessageReceiver.java | 9 +- .../db/changelog/db.changelog-tenant.yaml | 4 +- ...null-fields-in-manual-redaction-table.yaml | 37 ++++++ .../src/main/resources/application-dev.yaml | 2 +- .../analysislog/entitylog/EntityLogEntry.java | 2 - publish-custom-image.sh | 2 +- 16 files changed, 233 insertions(+), 46 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java index f85573a13..c9b64969b 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java @@ -1,10 +1,19 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse; + +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import java.util.HashMap; @@ -13,12 +22,19 @@ import java.util.Map; import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*; @RestController +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @RequiredArgsConstructor public class MigrationStatusController implements MigrationStatusResource { - private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; + SaasMigrationService saasMigrationService; + + SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; + + FileStatusService fileStatusService; + public MigrationStatusResponse migrationStatus() { + int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll(); Map filesInStatus = new HashMap<>(); @@ -36,10 +52,40 @@ public class MigrationStatusController implements MigrationStatusResource { errorCauses.put(errorFile.getFileId(), errorFile.getErrorCause()); }); - return MigrationStatusResponse.builder() - .numberOfFilesToMigrate(numberOfFilesToMigrate) - .filesInStatus(filesInStatus) - .errorCauses(errorCauses) - .build(); + return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build(); } + + + @Override + public ResponseEntity startMigrationForFile(String dossierId, String fileId) { + + if (!fileStatusService.fileExists(fileId)) { + throw new NotFoundException(String.format("File with id %s does not exist", fileId)); + } + + saasMigrationService.startMigrationForFile(dossierId, fileId); + + return ResponseEntity.ok().build(); + } + + + @Override + public ResponseEntity requeueErrorFiles() { + + MigrationStatusResponse migrationStatus = migrationStatus(); + if (!migrationIsFinished(migrationStatus)) { + throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!"); + } + + saasMigrationService.requeueErrorFiles(); + + return ResponseEntity.ok().build(); + } + + + private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) { + + return migrationStatus.getFilesInStatus().entrySet().stream().filter(e -> e.getValue() > 0).allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR)); + } + } diff --git a/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts index 952d22f49..fabf4463c 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts @@ -5,7 +5,7 @@ plugins { dependencies { api(project(":persistence-service-internal-api-v1")) api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.177.0") + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") api("com.iqser.red.service:search-service-api-v1:2.71.0") api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.4") diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java index a46bc2e81..728a4d532 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java @@ -5,13 +5,22 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; public interface MigrationStatusResource { String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status"; + String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration"; + String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration"; + String FILE_ID = "fileId"; + String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; + + String DOSSIER_ID = "dossierId"; + String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}"; @ResponseBody @PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE) @@ -19,5 +28,19 @@ public interface MigrationStatusResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) MigrationStatusResponse migrationStatus(); + @ResponseBody + @PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE) + @Operation(summary = "Start SAAS migration for specific file", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) + ResponseEntity startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId); + + @ResponseBody + @PostMapping(value = RETRY_MIGRATION_REST_PATH) + @Operation(summary = "Restart SAAS migration for all files in error state", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) + ResponseEntity requeueErrorFiles(); + + + } diff --git a/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts index 1b4f5f56e..5e0b5fa5c 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts @@ -7,7 +7,7 @@ dependencies { api(project(":persistence-service-external-api-v1")) api(project(":persistence-service-internal-api-v1")) api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.177.0") + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") api("com.iqser.red.service:search-service-api-v1:2.71.0") api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0") diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 08975e312..9ddc51767 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -13,7 +13,7 @@ dependencies { api("com.knecon.fforesight:keycloak-commons:0.22.0") api("com.knecon.fforesight:swagger-commons:0.5.0") api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.177.0") + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") api("com.knecon.fforesight:layoutparser-service-internal-api:0.74.0") api("com.iqser.red.service:search-service-api-v1:2.71.0") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java index 5ca7db547..5d30ca068 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java @@ -368,8 +368,7 @@ public class MessagingConfiguration { @Bean public Queue layoutparsingResponseQueue() { - return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); + return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 368eb30de..11327ffc3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -5,6 +5,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; @@ -19,8 +21,12 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService; import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent; import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; + import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; @@ -32,22 +38,23 @@ import static com.knecon.fforesight.service.layoutparser.internal.api.queue.Layo @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class SaasMigrationService implements TenantSyncService { - private final AutomaticAnalysisJob automaticAnalysisJob; - private final FileStatusPersistenceService fileStatusPersistenceService; - private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; - private final DossierService dossierService; + AutomaticAnalysisJob automaticAnalysisJob; + FileStatusPersistenceService fileStatusPersistenceService; + SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; + DossierService dossierService; + ManualRedactionProviderService manualRedactionProviderService; - private final LayoutParsingRequestFactory layoutParsingRequestFactory; - private final RabbitTemplate rabbitTemplate; - private final FileManagementServiceSettings settings; - private final StorageService storageService; + LayoutParsingRequestFactory layoutParsingRequestFactory; + RabbitTemplate rabbitTemplate; + FileManagementServiceSettings settings; + StorageService storageService; + SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; - private final SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; - - private final UncompressedFilesMigrationService uncompressedFilesMigrationService; + UncompressedFilesMigrationService uncompressedFilesMigrationService; @Override @@ -58,6 +65,7 @@ public class SaasMigrationService implements TenantSyncService { // Persistence-Service needs to be scaled to 1. + public void startMigrationForTenant(String tenantId) { // TODO migrate rules. @@ -79,22 +87,71 @@ public class SaasMigrationService implements TenantSyncService { } } - log.info("Added {} for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); + log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); + } + + + public void startMigrationForFile(String dossierId, String fileId) { + + log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId); + saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId); + var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false); + rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); + } + + + public void requeueErrorFiles() { + + automaticAnalysisJob.stopForTenant(TenantContext.getTenantId()); + saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR) + .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId())); + } public void handleLayoutParsingFinished(String dossierId, String fileId) { - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED); - rabbitTemplate.convertAndSend(MIGRATION_QUEUE, MigrationRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .build()); - log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); + if (!layoutParsingFilesExist(dossierId, fileId)) { + saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!"); + return; + } + + saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED); + try { + String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId(); + rabbitTemplate.convertAndSend(MIGRATION_QUEUE, + MigrationRequest.builder() + .dossierTemplateId(dossierTemplateId) + .dossierId(dossierId) + .fileId(fileId) + .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) + .build()); + log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); + } catch (Exception e) { + log.error("Queuing of entityLog migration failed with {}", e.getMessage()); + saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage())); + + } + + } + + + private boolean layoutParsingFilesExist(String dossierId, String fileId) { + + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) && storageService.objectExists( + TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) && storageService.objectExists(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) && storageService.objectExists(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); } public void handleEntityLogMigrationFinished(String dossierId, String fileId) { + + if (!entityLogMigrationFilesExist(dossierId, fileId)) { + saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!"); + return; + } saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED); log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); @@ -102,15 +159,21 @@ public class SaasMigrationService implements TenantSyncService { } + private boolean entityLogMigrationFilesExist(String dossierId, String fileId) { + + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists( + TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS)); + } + + public void handleError(String dossierId, String fileId, String errorCause, String retryQueue) { + var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId); Integer numErrors = migrationEntry.getProcessingErrorCounter(); - if (numErrors !=null && numErrors <= settings.getMaxErrorRetries()) { + if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) { saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause); - rabbitTemplate.convertAndSend(retryQueue, MigrationRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .build()); + rabbitTemplate.convertAndSend(retryQueue, MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build()); log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause); } else { saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause); @@ -127,7 +190,7 @@ public class SaasMigrationService implements TenantSyncService { saasAnnotationIdMigrationService.updateAnnotationIds(fileId, oldToNewMapping); } catch (Exception e) { saasMigrationStatusPersistenceService.updateErrorStatus(fileId, e.getMessage()); - log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, e.getMessage()); + log.error("Error during annotation id migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, e.getMessage()); throw e; } saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); @@ -151,7 +214,4 @@ public class SaasMigrationService implements TenantSyncService { } } - - - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index 4324f5000..8d76f335a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -96,7 +96,7 @@ public class EntityLogMergeService { boolean isHint = isHint(manualRedactionEntry.getType(), dossier); - if (manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE)) { + if (isFalsePositive(manualRedactionEntry)) { var matchingEntities = entityLog.getEntityLogEntry().stream() .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) .toList(); @@ -146,6 +146,12 @@ public class EntityLogMergeService { } + private static boolean isFalsePositive(ManualRedactionEntry manualRedactionEntry) { + + return manualRedactionEntry.getDictionaryEntryType() != null && manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE); + } + + private void mergeFalsePositive(EntityLog entityLog, EntityLogEntry existingEntry) { existingEntry.setState(EntryState.REMOVED); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 74cd03b42..7b2fcd128 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -130,6 +130,11 @@ public class FileStatusService { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(converted); } + public boolean fileExists(String fileId) { + + return fileStatusPersistenceService.statusExists(fileId); + } + @Transactional public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 776ad6289..87f8a2261 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -560,4 +560,10 @@ public class FileStatusPersistenceService { return fileRepository.getFilenameById(fileId).orElseThrow(); } + + public boolean statusExists(String fileId) { + + return fileRepository.existsById(fileId); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index fb43103a4..c9b85a19b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; @@ -11,9 +12,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; + import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; + import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.stereotype.Service; @@ -58,13 +61,15 @@ public class LayoutParsingFinishedMessageReceiver { @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_DLQ) public void handleDLQMessage(Message failedMessage) { - var analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class); + LayoutParsingRequest analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class); log.info("Failed to process analyze request: {}", analyzeRequest); String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER); if (saasMigrationStatusPersistenceService.isMigrating(layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()))) { saasMigrationService.handleError(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()), - layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()), errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE); + layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()), + errorCause, + LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE); return; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index fabc20128..6829aa72a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -168,4 +168,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/114-add-download-redaction-file-status-table.yaml - include: - file: db/changelog/tenant/115-add-saas-migration-status-table.yaml \ No newline at end of file + file: db/changelog/tenant/115-add-saas-migration-status-table.yaml + - include: + file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml new file mode 100644 index 000000000..f71cce9c1 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml @@ -0,0 +1,37 @@ +databaseChangeLog: + - changeSet: + id: makeFieldsNonNullable + author: your_author_name + changes: + - addNotNullConstraint: + columnName: add_to_dictionary + tableName: manual_redaction + defaultNullValue: 'false' + - addNotNullConstraint: + columnName: add_to_all_dossiers + tableName: manual_redaction + defaultNullValue: 'true' + - addNotNullConstraint: + columnName: add_to_dictionary + tableName: manual_recategorization + defaultNullValue: 'false' + - addNotNullConstraint: + columnName: add_to_all_dossiers + tableName: manual_recategorization + defaultNullValue: 'true' + - addNotNullConstraint: + columnName: update_dictionary + tableName: manual_resize_redaction + defaultNullValue: 'false' + - addNotNullConstraint: + columnName: add_to_all_dossiers + tableName: manual_resize_redaction + defaultNullValue: 'true' + - addNotNullConstraint: + columnName: remove_from_dictionary + tableName: id_removal + defaultNullValue: 'false' + - addNotNullConstraint: + columnName: remove_from_all_dossiers + tableName: id_removal + defaultNullValue: 'true' diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml index 0c80ee4d2..451cedd61 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml @@ -35,7 +35,7 @@ persistence-service: applicationName: RedactManager fforesight: springdoc: - auth-server-url: 'http://localhost:8080/auth' + auth-server-url: 'http://localhost:8080' jobs: enabled: true datasource: diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogEntry.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogEntry.java index b8105560c..00a6058d9 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogEntry.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntityLogEntry.java @@ -36,8 +36,6 @@ public class EntityLogEntry { String closestHeadline; String section; - float[] color; - @Builder.Default List positions = new ArrayList<>(); diff --git a/publish-custom-image.sh b/publish-custom-image.sh index 89b9f8d94..b680eb16a 100755 --- a/publish-custom-image.sh +++ b/publish-custom-image.sh @@ -11,5 +11,5 @@ commit_hash=$(git rev-parse --short=5 HEAD) # Combine branch and commit hash buildName="${USER}-${branch}-${commit_hash}" -gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName +gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"