From ce399685083c75515603fb2cefd667401025818a Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 12 Feb 2024 15:47:41 +0100 Subject: [PATCH] RED-7384: fix db fields for manual changes and tenant details --- .../migration/SaasMigrationService.java | 50 +++++++++++------- .../AnalysisFlagsCalculationService.java | 52 +++++++++++-------- .../download/DownloadCompressingService.java | 5 +- .../service/job/DownloadReadyJob.java | 33 +++++++----- .../v1/processor/utils/TenantUtils.java | 6 +-- .../db/changelog/db.changelog-tenant.yaml | 2 + ...s-correctly-in-manual-redaction-table.yaml | 33 ++++++++++++ 7 files changed, 120 insertions(+), 61 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 0d24b2078..014d0e06f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -23,6 +23,8 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService; import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent; import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.TenantProvider; +import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -50,14 +52,12 @@ public class SaasMigrationService implements TenantSyncService { SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; DossierService dossierService; ManualRedactionProviderService manualRedactionProviderService; - + TenantProvider tenantProvider; LayoutParsingRequestFactory layoutParsingRequestFactory; RabbitTemplate rabbitTemplate; FileManagementServiceSettings settings; StorageService storageService; - SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; - UncompressedFilesMigrationService uncompressedFilesMigrationService; @@ -80,9 +80,15 @@ public class SaasMigrationService implements TenantSyncService { log.info("Finished uncompressed files migration ..."); int numberOfFiles = 0; - var dossiers = dossierService.getAllDossiers().stream().filter(dossier -> dossier.getHardDeletedTime() == null).toList(); + var dossiers = dossierService.getAllDossiers() + .stream() + .filter(dossier -> dossier.getHardDeletedTime() == null) + .toList(); for (var dossier : dossiers) { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()).stream().filter(file -> file.getHardDeletedTime() == null).toList(); + var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()) + .stream() + .filter(file -> file.getHardDeletedTime() == null) + .toList(); var migrationStati = saasMigrationStatusPersistenceService.findAll() .stream() .collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus)); @@ -143,12 +149,12 @@ public class SaasMigrationService implements TenantSyncService { try { String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId(); rabbitTemplate.convertAndSend(MIGRATION_QUEUE, - MigrationRequest.builder() - .dossierTemplateId(dossierTemplateId) - .dossierId(dossierId) - .fileId(fileId) - .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) - .build()); + MigrationRequest.builder() + .dossierTemplateId(dossierTemplateId) + .dossierId(dossierId) + .fileId(fileId) + .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) + .build()); log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); } catch (Exception e) { log.error("Queuing of entityLog migration failed with {}", e.getMessage()); @@ -160,11 +166,14 @@ public class SaasMigrationService implements TenantSyncService { private boolean layoutParsingFilesExist(String dossierId, String fileId) { - return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) && storageService.objectExists( - TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) && storageService.objectExists(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) && storageService.objectExists(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) + && storageService.objectExists(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) + && storageService.objectExists(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, + fileId, + FileType.DOCUMENT_PAGES)) + && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); } @@ -232,10 +241,10 @@ public class SaasMigrationService implements TenantSyncService { updateAnnotationIds(fileId, idMapping); } catch (Exception e) { String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s", - TenantContext.getTenantId(), - dossierId, - fileId, - e.getMessage()); + TenantContext.getTenantId(), + dossierId, + fileId, + e.getMessage()); saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message); log.error(message); throw e; @@ -247,6 +256,7 @@ public class SaasMigrationService implements TenantSyncService { if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) { automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); + tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build()); log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId()); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java index 22326108d..445d34b2a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java @@ -51,7 +51,8 @@ public class AnalysisFlagsCalculationService { var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee()); - Map viewedPages = viewedPagesForCurrentAssignee.stream().collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime)); + Map viewedPages = viewedPagesForCurrentAssignee.stream() + .collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime)); boolean hasRedactions = false; boolean hasHints = false; @@ -67,7 +68,8 @@ public class AnalysisFlagsCalculationService { String type = getType(entry.getType()); - var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges().get(entry.getChanges().size() - 1); + var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges() + .get(entry.getChanges().size() - 1); var entryType = entry.getEntryType(); if (entry.getManualChanges() != null && !entry.getManualChanges().isEmpty()) { @@ -87,13 +89,14 @@ public class AnalysisFlagsCalculationService { } if (lastChange != null && // - (lastRedactionModification == null || lastChange.getDateTime().isAfter(lastRedactionModification)) && // - !entryType.equals(EntryType.HINT) && !entryType.equals(EntryType.RECOMMENDATION)) { + (lastRedactionModification == null || lastChange.getDateTime().isAfter(lastRedactionModification)) && // + !entryType.equals(EntryType.HINT) &&// + !entryType.equals(EntryType.RECOMMENDATION)) { lastRedactionModification = lastChange.getDateTime(); } if (!hasRedactions && entry.getState().equals(EntryState.APPLIED) && // - (entryType.equals(EntryType.ENTITY) || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.AREA))) { + (entryType.equals(EntryType.ENTITY) || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.AREA))) { hasRedactions = true; } @@ -105,23 +108,28 @@ public class AnalysisFlagsCalculationService { hasImages = true; } - OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions().get(0).getPageNumber()); + OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions() + .get(0).getPageNumber()); if (file.getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && // - viewedPage.isBefore(lastChange.getDateTime())) { + viewedPage.isBefore(lastChange.getDateTime())) { hasUpdates = true; } } log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime); - if (file.isHasRedactions() == hasRedactions && file.isHasHints() == hasHints && file.isHasImages() == hasImages && file.isHasSuggestions() == hasSuggestions && file.isHasAnnotationComments() == hasComments && file.isHasUpdates() == hasUpdates) { + if (file.isHasRedactions() == hasRedactions + && file.isHasHints() == hasHints + && file.isHasImages() == hasImages + && file.isHasSuggestions() == hasSuggestions + && file.isHasAnnotationComments() == hasComments + && file.isHasUpdates() == hasUpdates) { log.info("Nothing Changed for file: {}", fileId); } else { fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); } - if (lastRedactionModification != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate().isBefore(lastRedactionModification))) { fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, lastRedactionModification); } @@ -136,9 +144,7 @@ public class AnalysisFlagsCalculationService { private void addIdsToTrace(String dossierId, String fileId) { if (observationRegistry.getCurrentObservation() != null) { - observationRegistry.getCurrentObservation() - .highCardinalityKeyValue("dossierId", dossierId) - .highCardinalityKeyValue("fileId", fileId); + observationRegistry.getCurrentObservation().highCardinalityKeyValue("dossierId", dossierId).highCardinalityKeyValue("fileId", fileId); } } @@ -149,17 +155,17 @@ public class AnalysisFlagsCalculationService { OffsetDateTime lastRedactionModification) { return !entryType.equals(EntryType.HINT) && // - !entryType.equals(EntryType.RECOMMENDATION) && // - StringUtils.isNotEmpty(entry.getReason()) && // - (manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || // - manualChange.getManualRedactionType().equals(ManualRedactionType.RESIZE)) && // - manualChange.getProcessedDate() != null && // - (lastRedactionModification == null || manualChange.getProcessedDate().isAfter(lastRedactionModification)); + !entryType.equals(EntryType.RECOMMENDATION) && // + StringUtils.isNotEmpty(entry.getReason()) && // + (manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || // + manualChange.getManualRedactionType().equals(ManualRedactionType.RESIZE)) && // + manualChange.getProcessedDate() != null && // + (lastRedactionModification == null || manualChange.getProcessedDate().isAfter(lastRedactionModification)); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java index 253315d88..eaf6fcfa4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java @@ -7,8 +7,11 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; -import lombok.RequiredArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j @Service @RequiredArgsConstructor public class DownloadCompressingService { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java index ea8d44cfa..cacfa6587 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java @@ -8,8 +8,10 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUti import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.stereotype.Service; @@ -30,25 +32,28 @@ public class DownloadReadyJob implements Job { log.debug("Running DownloadReadyJob"); - tenantProvider.getTenants().forEach(tenant -> { + tenantProvider.getTenants() + .forEach(tenant -> { - if (!TenantUtils.isTenantReadyForPersistence(tenant)) { - return; - } + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { + return; + } - TenantContext.setTenantId(tenant.getTenantId()); + TenantContext.setTenantId(tenant.getTenantId()); - downloadStatusPersistenceService.getStatus().stream().filter(d -> d.getStatus().equals(DownloadStatusValue.GENERATING)).forEach(download -> { + downloadStatusPersistenceService.getStatus() + .stream() + .filter(d -> d.getStatus().equals(DownloadStatusValue.GENERATING)) + .forEach(download -> { - int numberOfFiles = download.getFiles().size(); - var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId()); - if (downloadRedactionFileStatus.size() == numberOfFiles) { - downloadCompressingService.markDownloadForCompression(download.getStorageId(),download.getUserId()); - } - }); - }); + int numberOfFiles = download.getFiles().size(); + var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId()); + if (downloadRedactionFileStatus.size() >= numberOfFiles) { + downloadCompressingService.markDownloadForCompression(download.getStorageId(), download.getUserId()); + } + }); + }); } - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/TenantUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/TenantUtils.java index 1d5637559..acf72bcb4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/TenantUtils.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/TenantUtils.java @@ -4,9 +4,9 @@ import com.knecon.fforesight.tenantcommons.model.TenantResponse; public class TenantUtils { - public static boolean isTenantReadyForPersistence(TenantResponse tenantResponse){ - return tenantResponse.getDetails()!=null && - tenantResponse.getDetails().getOrDefault("persistence-service-ready",false).equals(true); + public static boolean isTenantReadyForPersistence(TenantResponse tenantResponse) { + + return tenantResponse.getDetails() != null && tenantResponse.getDetails().getOrDefault("persistence-service-ready", false).equals(true); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 0ae66685f..ba852e780 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -179,3 +179,5 @@ databaseChangeLog: file: db/changelog/tenant/117-add-last-flag-calculation-date-to-file.yaml - include: file: db/changelog/tenant/118-delete-unapproved-manual-changes-and-drop-column-status.yaml + - include: + file: db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml new file mode 100644 index 000000000..30f0013bb --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml @@ -0,0 +1,33 @@ +databaseChangeLog: + - changeSet: + id: setAddToAllDossiersFalse + author: your_username + changes: + - sql: + dbms: postgresql + endDelimiter: ';' + sql: | + UPDATE manual_redaction + SET add_to_all_dossiers = false + WHERE add_to_dictionary = false; + - sql: + dbms: postgresql + endDelimiter: ';' + sql: | + UPDATE manual_recategorization + SET add_to_all_dossiers = false + WHERE add_to_dictionary = false; + - sql: + dbms: postgresql + endDelimiter: ';' + sql: | + UPDATE manual_resize_redaction + SET add_to_all_dossiers = false + WHERE update_dictionary = false; + - sql: + dbms: postgresql + endDelimiter: ';' + sql: | + UPDATE id_removal + SET remove_from_all_dossiers = false + WHERE remove_from_dictionary = false;