From 1fc74f2a0cfe4a604e2176aa53f5f6d107554b0c Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 3 Apr 2024 16:30:58 +0200 Subject: [PATCH 1/4] RED-7384: migration fixes forward port * treat pending dicts in approved files for resize and remove in redaction-service * disable automatic analysis job using migration status table and liquibase migration * migrate imported redaction file * improve logging --- ...er.red.service.java-conventions.gradle.kts | 3 +- .../service/CustomPermissionService.java | 4 +- .../migration/SaasMigrationService.java | 87 +++++++++---------- .../MigrateImportedRedactionsFiles17.java | 84 ++++++++++++++++++ .../service/KeyCloakUserSyncService.java | 2 +- .../ReanalysisRequiredStatusService.java | 3 +- .../service/job/AutomaticAnalysisJob.java | 21 ++++- ...SaasMigrationStatusPersistenceService.java | 5 ++ .../SaasMigrationStatusRepository.java | 4 + ...tion-required-status-for-each-present-file | 10 +++ 10 files changed, 168 insertions(+), 55 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index a84a8579f..176517d6c 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -6,7 +6,7 @@ plugins { jacoco } -val redactionServiceVersion by rootProject.extra { "4.262.0" } +val redactionServiceVersion by rootProject.extra { "4.290.0" } val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" } val redactionReportServiceVersion by rootProject.extra { "4.47.0" } val searchServiceVersion by rootProject.extra { "2.71.0" } @@ -49,7 +49,6 @@ tasks.named("test") { tasks.test { finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run - } tasks.jacocoTestReport { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/service/CustomPermissionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/service/CustomPermissionService.java index 2828058a0..bf7a5b55f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/service/CustomPermissionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/service/CustomPermissionService.java @@ -40,10 +40,10 @@ public class CustomPermissionService { public void syncAllCustomPermissions() { - log.info("Syncing all custom permissions"); + log.debug("Syncing all custom permissions"); var targetObjects = getAllSupportedTargetObjects(); targetObjects.forEach(this::applyCustomPermissions); - log.info("All custom permissions synced"); + log.debug("All custom permissions synced"); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 6df54ba82..cc0339fdd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -1,24 +1,21 @@ package com.iqser.red.service.persistence.management.v1.processor.migration; import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE; -import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType.ENTITY_LOG; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; +import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob; @@ -69,7 +66,9 @@ public class SaasMigrationService implements TenantSyncService { SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; UncompressedFilesMigrationService uncompressedFilesMigrationService; ManualRedactionService manualRedactionService; + CommentService commentService; RankDeDuplicationService rankDeDuplicationService; + SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService; @Override @@ -91,48 +90,29 @@ public class SaasMigrationService implements TenantSyncService { log.info("Finished uncompressed files migration ..."); rankDeDuplicationService.deduplicate(); - int numberOfFiles = 0; - var dossiers = dossierService.getAllDossiers() - .stream() - .filter(dossier -> dossier.getHardDeletedTime() == null) - .toList(); - for (var dossier : dossiers) { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()) - .stream() - .filter(file -> file.getHardDeletedTime() == null) - .toList(); + var files = saasMigrationStatusPersistenceService.findAll(); - var migrationStati = saasMigrationStatusPersistenceService.findAll() - .stream() - .collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus)); + for (var file : files) { - for (var file : files) { - if (notExistsOrError(file, migrationStati)) { - // delete NER_ENTITIES since offsets depend on old document structure. - storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossier.getId(), file.getId(), FileType.NER_ENTITIES)); - saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId()); - var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false); - rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); - numberOfFiles++; - } else { - log.info("Skipping file with id {} and dossier {}, since it is already migrated or is currently being migrated!", file.getId(), file.getDossierId()); - } + if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) { + log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus()); + continue; } + + // delete NER_ENTITIES since offsets depend on old document structure. + storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES)); + + var layoutParsingRequest = layoutParsingRequestFactory.build(file.getDossierId(), file.getFileId(), false); + + rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); + + numberOfFiles++; + } log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); - - if (numberOfFiles == 0) { - finalizeMigration(); - } - } - - - private boolean notExistsOrError(FileEntity file, Map migrationStati) { - - return migrationStati.getOrDefault(file.getId(), SaasMigrationStatus.ERROR).equals(SaasMigrationStatus.ERROR); } @@ -161,24 +141,27 @@ public class SaasMigrationService implements TenantSyncService { return; } + log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED); if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { - manualRedactionProviderService.convertUnprocessedAddToDictionariesToLocalChanges(fileId); + saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId); } try { indexingService.reindex(dossierId, Set.of(fileId), false); String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId(); + rabbitTemplate.convertAndSend(MIGRATION_QUEUE, MigrationRequest.builder() .dossierTemplateId(dossierTemplateId) .dossierId(dossierId) .fileId(fileId) + .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) + .entitiesWithComments(commentService.getCommentCounts(fileId).keySet()) .build()); - log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); } catch (Exception e) { log.error("Queuing of entityLog migration failed with {}", e.getMessage()); saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage())); @@ -211,8 +194,9 @@ public class SaasMigrationService implements TenantSyncService { private boolean entityLogMigrationFilesExist(String dossierId, String fileId) { - return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, ENTITY_LOG)) - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS)); + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists( + TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS)); } @@ -239,26 +223,39 @@ public class SaasMigrationService implements TenantSyncService { List manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd(); int count = addManualRedactionEntries(manualRedactionEntriesToAdd); log.info("Added {} additional manual entries.", count); - deleteSectionGrid(dossierId, fileId); + deleteSectionGridAndNerEntitiesFiles(dossierId, fileId); saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); - finalizeMigration(); +// finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change +// This ensures no analysis will happen with outdated rules } - private void deleteSectionGrid(String dossierId, String fileId) { + private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) { try { storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)); } catch (StorageObjectDoesNotExist e) { log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId); } + + try { + storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); + } catch (StorageObjectDoesNotExist e) { + log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId); + } } private int addManualRedactionEntries(List manualRedactionEntriesToAdd) { + manualRedactionEntriesToAdd.forEach(add -> { + if (add.getSection() != null && add.getSection().length() > 254) { + add.setSection(add.getSection().substring(0, 254)); + } + }); + return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java new file mode 100644 index 000000000..f6ed7ffe8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java @@ -0,0 +1,84 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import java.util.Collection; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Setter +@Service +public class MigrateImportedRedactionsFiles17 extends Migration { + + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + + @Autowired + private FileManagementStorageService fileManagementStorageService; + + private static final String NAME = "Migrating the ImportedRedaction files to the new structure"; + private static final long VERSION = 17; + + + public MigrateImportedRedactionsFiles17() { + + super(NAME, VERSION); + } + + + @Override + protected void migrate() { + + var files = fileStatusPersistenceService.getAllFiles(); + for (FileEntity file : files) { + + if (fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS)) { + + com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getImportedRedactions( + file.getDossierId(), + file.getId()); + + fileManagementStorageService.deleteObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS); + + List importedRedactionList = oldImportedRedactions.getImportedRedactions().values() + .stream() + .flatMap(Collection::stream) + .map(this::toImportedRedaction) + .toList(); + + ImportedRedactions importedRedactions = new ImportedRedactions(importedRedactionList); + + fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS, importedRedactions); + } + } + } + + + private ImportedRedaction toImportedRedaction(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction importedRedaction) { + + return ImportedRedaction.builder().id(importedRedaction.getId()).positions(mapPositions(importedRedaction)).build(); + } + + + private static List mapPositions(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction importedRedaction) { + + return importedRedaction.getPositions() + .stream() + .map(rectangle -> new Position(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())) + .toList(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/KeyCloakUserSyncService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/KeyCloakUserSyncService.java index a3e779ffe..a3dc4eb2e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/KeyCloakUserSyncService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/KeyCloakUserSyncService.java @@ -57,7 +57,7 @@ public class KeyCloakUserSyncService { // remove KC users, what's left is users that are in redaction but no longer in KC redactionObjectsUserIds.removeAll(allUserIds); - log.info("Performing user sync/cleanup for ids: {}", redactionObjectsUserIds); + log.debug("Performing user sync/cleanup for ids: {}", redactionObjectsUserIds); redactionObjectsUserIds.forEach(removedUser -> alLDossiers.forEach(dossier -> this.userService.updateDossierUsers(removedUser, UserService.UserRemovalModel.PERMANENT, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java index e42a001c0..cb1a65a24 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java @@ -132,7 +132,8 @@ public class ReanalysisRequiredStatusService { var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches; if (reanalysisRequired || fullAnalysisRequired) { log.info( - "For file: {} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", + "For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", + fileStatus.getId(), fileStatus.getFilename(), fileStatus.getRulesVersion(), dossierTemplateVersions.getOrDefault(RULES, -1L), diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index 26a23fc80..ebc985b83 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; @@ -34,6 +35,7 @@ public class AutomaticAnalysisJob implements Job { private final FileStatusService fileStatusService; private final TenantProvider tenantProvider; private final ObservationRegistry observationRegistry; + private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; @Setter private boolean schedulingStopped; @@ -52,13 +54,22 @@ public class AutomaticAnalysisJob implements Job { tenantProvider.getTenants() .forEach(tenant -> { - if (!TenantUtils.isTenantReadyForPersistence(tenant) || stoppedTenants.contains(tenant.getTenantId())) { + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { + log.info("[Tenant:{}] Skipping scheduling since tenant is not ready.", tenant.getTenantId()); + return; + } + + if (stoppedTenants.contains(tenant.getTenantId())) { + log.info("[Tenant:{}] Skipping scheduling as automatic reanalysis is disabled for tenant.", tenant.getTenantId()); return; } TenantContext.setTenantId(tenant.getTenantId()); - var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); + if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) { + log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId()); + return; + }var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", tenant.getTenantId(), @@ -107,10 +118,10 @@ public class AutomaticAnalysisJob implements Job { .observe(() -> { if (file.isFullAnalysisRequired()) { - log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getFilename()); + log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getId()); fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false); } else if (file.isReanalysisRequired()) { - log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getFilename()); + log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getId()); fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false); } }); @@ -125,12 +136,14 @@ public class AutomaticAnalysisJob implements Job { public void stopForTenant(String tenantId) { + log.info("Stopping automatic analysis for tenant {}", tenantId); stoppedTenants.add(tenantId); } public void startForTenant(String tenantId) { + log.info("Starting automatic analysis for tenant {}", tenantId); stoppedTenants.remove(tenantId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java index b38dd95c7..a2a71292e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java @@ -42,6 +42,11 @@ public class SaasMigrationStatusPersistenceService { return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED; } + public boolean migrationFinishedForTenant() { + + return saasMigrationStatusRepository.findAllWhereStatusNotFinishedAndNotError() == 0; + } + @Transactional public void createMigrationRequiredStatus(String dossierId, String fileId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java index 3265ea355..e6ced6cea 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java @@ -37,4 +37,8 @@ public interface SaasMigrationStatusRepository extends JpaRepository Date: Wed, 3 Apr 2024 17:23:57 +0200 Subject: [PATCH 2/4] RED-7384: migration fixes * finalize migration again for "persistence-service-ready": true details --- ...asMigrationManualChangesUpdateService.java | 83 +++++++++++++++++++ .../migration/SaasMigrationService.java | 5 +- 2 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java new file mode 100644 index 000000000..cfdcfbd6f --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java @@ -0,0 +1,83 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; + +import org.springframework.stereotype.Service; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class SaasMigrationManualChangesUpdateService { + + private final AddRedactionPersistenceService addRedactionPersistenceService; + + private final HashFunction hashFunction = Hashing.murmur3_128(); + + + public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) { + + var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); + for (var unprocessedManualAdd : unprocessedManualAdds) { + + if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) { + continue; + } + + if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) { + // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table. + // It has no functionality, its only there, such that the UI can show a pending change. + ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder() + .id(buildSecondaryId(unprocessedManualAdd.getId(), fileId)) + .user(unprocessedManualAdd.getUser()) + .typeId(unprocessedManualAdd.getTypeId()) + .value(unprocessedManualAdd.getValue()) + .reason(unprocessedManualAdd.getReason()) + .legalBasis(unprocessedManualAdd.getLegalBasis()) + .section(unprocessedManualAdd.getSection()) + .rectangle(unprocessedManualAdd.isRectangle()) + .addToDictionary(unprocessedManualAdd.isAddToDictionary()) + .addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers()) + .dictionaryEntryType(DictionaryEntryType.ENTRY) + .requestDate(unprocessedManualAdd.getRequestDate()) + .positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List + .fileStatus(unprocessedManualAdd.getFileStatus()) + .textBefore(unprocessedManualAdd.getTextBefore()) + .textAfter(unprocessedManualAdd.getTextAfter()) + .sourceId(unprocessedManualAdd.getSourceId()) + .typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()) + .build(); + + addRedactionPersistenceService.update(pendingDictAdd); + + // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it. + unprocessedManualAdd.setAddToDictionary(false); + unprocessedManualAdd.setAddToAllDossiers(false); + unprocessedManualAdd.setLegalBasis(""); + unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet()); + unprocessedManualAdd.setDictionaryEntryType(null); + + addRedactionPersistenceService.update(unprocessedManualAdd); + } + } + } + + + private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) { + + return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index cc0339fdd..30f299e4c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -227,8 +227,7 @@ public class SaasMigrationService implements TenantSyncService { saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); -// finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change -// This ensures no analysis will happen with outdated rules + finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change } @@ -297,7 +296,7 @@ public class SaasMigrationService implements TenantSyncService { private void finalizeMigration() { if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) { - automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); +// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build()); log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId()); } From 910948bd2de4a43b6e953539c9b9ae6806c5b0ce Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 3 Apr 2024 17:35:38 +0200 Subject: [PATCH 3/4] RED-7384: migration fixes * finalize migration again for "persistence-service-ready": true details --- .../src/main/resources/db/changelog/db.changelog-tenant.yaml | 2 ++ ...create-migration-required-status-for-each-present-file.yaml} | 0 2 files changed, 2 insertions(+) rename persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/{124-create-migration-required-status-for-each-present-file => 124-create-migration-required-status-for-each-present-file.yaml} (100%) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 5da7207a3..e649a1f3d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -193,3 +193,5 @@ databaseChangeLog: file: db/changelog/tenant/122-add-legal-basis-variables-to-recategorize.yaml - include: file: db/changelog/tenant/123-add-value-to-recategorize.yaml + - include: + file: db/changelog/tenant/124-create-migration-required-status-for-each-present-file.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file.yaml similarity index 100% rename from persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file rename to persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file.yaml From 88d63b46b9d3ea889792406f14539150d6e19f4e Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 4 Apr 2024 14:45:17 +0200 Subject: [PATCH 4/4] RED-7384: migration fixes * finalize migration again for "persistence-service-ready": true details --- .../MigrateImportedRedactionsFiles17.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java index f6ed7ffe8..1fbd780fb 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.management.v1.processor.migration.migr import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -13,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import lombok.Setter; @@ -53,13 +56,15 @@ public class MigrateImportedRedactionsFiles17 extends Migration { fileManagementStorageService.deleteObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS); - List importedRedactionList = oldImportedRedactions.getImportedRedactions().values() + Map> importedRedactionsPerPage = oldImportedRedactions.getImportedRedactions().entrySet() .stream() - .flatMap(Collection::stream) - .map(this::toImportedRedaction) - .toList(); + .collect(Collectors.toMap(Map.Entry::getKey, + entry -> entry.getValue() + .stream() + .map(this::toImportedRedaction) + .toList())); - ImportedRedactions importedRedactions = new ImportedRedactions(importedRedactionList); + ImportedRedactionsPerPage importedRedactions = new ImportedRedactionsPerPage(importedRedactionsPerPage); fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS, importedRedactions); }