diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java deleted file mode 100644 index 917bd8b2f..000000000 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.iqser.red.persistence.service.v1.external.api.impl.controller; - -import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse; - -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; - -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RestController; - -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*; - -@RestController -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -@RequiredArgsConstructor -public class MigrationStatusController implements MigrationStatusResource { - - SaasMigrationService saasMigrationService; - - SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; - - FileStatusService fileStatusService; - - - public MigrationStatusResponse migrationStatus() { - - int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll(); - - Map filesInStatus = new HashMap<>(); - filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED)); - filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED)); - filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED)); - filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED)); - filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED)); - filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR)); - - var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR); - - var errorCauses = filesInErrorState.stream() - .collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause)); - - return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build(); - } - - - @Override - public ResponseEntity startMigrationForFile(String dossierId, String fileId) { - - if (!fileStatusService.fileExists(fileId)) { - throw new NotFoundException(String.format("File with id %s does not exist", fileId)); - } - - saasMigrationService.startMigrationForFile(dossierId, fileId); - - return ResponseEntity.ok().build(); - } - - - @Override - public ResponseEntity revertMigrationForFile(String dossierId, String fileId) { - - if (!fileStatusService.fileExists(fileId)) { - throw new NotFoundException(String.format("File with id %s does not exist", fileId)); - } - - if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) { - throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId)); - } - - saasMigrationService.revertMigrationForFile(dossierId, fileId); - - return ResponseEntity.ok().build(); - } - - - @Override - public ResponseEntity requeueErrorFiles() { - - MigrationStatusResponse migrationStatus = migrationStatus(); - if (!migrationIsFinished(migrationStatus)) { - throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!"); - } - - saasMigrationService.requeueErrorFiles(); - - return ResponseEntity.ok().build(); - } - - - private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) { - - return migrationStatus.getFilesInStatus().entrySet() - .stream() - .filter(e -> e.getValue() > 0) - .allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR)); - } - -} diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java deleted file mode 100644 index 6ec42c6a1..000000000 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.iqser.red.service.persistence.service.v1.api.external.resource; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; - -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; - -public interface MigrationStatusResource { - - String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status"; - String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration"; - String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration"; - String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration"; - - String FILE_ID = "fileId"; - String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; - - String DOSSIER_ID = "dossierId"; - String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}"; - - - @ResponseBody - @PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Show the status of the migration", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) - MigrationStatusResponse migrationStatus(); - - - @ResponseBody - @PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE) - @Operation(summary = "Start SAAS migration for specific file", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) - ResponseEntity startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId); - - - @ResponseBody - @PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE) - @Operation(summary = "Start SAAS migration for specific file", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) - ResponseEntity revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId); - - - @ResponseBody - @PostMapping(value = RETRY_MIGRATION_REST_PATH) - @Operation(summary = "Restart SAAS migration for all files in error state", description = "None") - @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")}) - ResponseEntity requeueErrorFiles(); - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java deleted file mode 100644 index b4ed82546..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.entity.migration; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; - -import jakarta.persistence.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@Entity -@Table(name = "saas_migration_status") -public class SaasMigrationStatusEntity { - - @Id - private String fileId; - - @Column - private String dossierId; - - @Column - @Enumerated(EnumType.STRING) - private SaasMigrationStatus status; - - @Column - private Integer processingErrorCounter; - - @Column - private String errorCause; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java deleted file mode 100644 index 239f2981f..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java +++ /dev/null @@ -1,170 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration; - -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*; -import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; - -import jakarta.transaction.Transactional; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Service; - -@Slf4j -@Service -@Transactional -@RequiredArgsConstructor -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class SaasAnnotationIdMigrationService { - - ManualRedactionRepository manualRedactionRepository; - RemoveRedactionRepository removeRedactionRepository; - ForceRedactionRepository forceRedactionRepository; - ResizeRedactionRepository resizeRedactionRepository; - RecategorizationRepository recategorizationRepository; - LegalBasisChangeRepository legalBasisChangeRepository; - CommentRepository commentRepository; - FileRepository fileRepository; - - - public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), ManualRedactionEntryEntity.class); - newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class)); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - newEntry.setId(newAnnotationEntityId); - - manualRedactionRepository.deleteById(oldAnnotationEntityId); - manualRedactionRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), IdRemovalEntity.class); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - newEntry.setId(newAnnotationEntityId); - - removeRedactionRepository.deleteById(oldAnnotationEntityId); - removeRedactionRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), ManualForceRedactionEntity.class); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - newEntry.setId(newAnnotationEntityId); - - forceRedactionRepository.deleteById(oldAnnotationEntityId); - forceRedactionRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), ManualResizeRedactionEntity.class); - newEntry.setId(newAnnotationEntityId); - newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class)); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - - resizeRedactionRepository.deleteById(oldAnnotationEntityId); - resizeRedactionRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), ManualRecategorizationEntity.class); - newEntry.setId(newAnnotationEntityId); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - - recategorizationRepository.deleteById(oldAnnotationEntityId); - recategorizationRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - - if (oldAnnotationEntityId.equals(newAnnotationEntityId)) { - return 0; - } - var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId); - if (oldEntry.isPresent()) { - - var newEntry = MagicConverter.convert(oldEntry.get(), ManualLegalBasisChangeEntity.class); - newEntry.setId(newAnnotationEntityId); - newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId()) - .get()); - - legalBasisChangeRepository.deleteById(oldAnnotationEntityId); - legalBasisChangeRepository.save(newEntry); - return 1; - } - return 0; - } - - - public int updateCommentIds(String fileId, String key, String value) { - - if (key.equals(value)) { - return 0; - } - return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java deleted file mode 100644 index ddb6ba85e..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; - -import org.springframework.stereotype.Service; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; - -import lombok.RequiredArgsConstructor; - -@Service -@RequiredArgsConstructor -public class SaasMigrationManualChangesUpdateService { - - private final AddRedactionPersistenceService addRedactionPersistenceService; - - private final HashFunction hashFunction = Hashing.murmur3_128(); - - - public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) { - - var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly()); - for (var unprocessedManualAdd : unprocessedManualAdds) { - - if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) { - continue; - } - - if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) { - // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table. - // It has no functionality, its only there, such that the UI can show a pending change. - ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId), - unprocessedManualAdd.getUser(), - unprocessedManualAdd.getTypeId(), - unprocessedManualAdd.getValue(), - unprocessedManualAdd.getReason(), - unprocessedManualAdd.getLegalBasis(), - unprocessedManualAdd.getSection(), - unprocessedManualAdd.isRectangle(), - unprocessedManualAdd.isAddToDictionary(), - unprocessedManualAdd.isAddToAllDossiers(), - unprocessedManualAdd.isAddToDossierDictionary(), - DictionaryEntryType.ENTRY, - unprocessedManualAdd.getRequestDate(), - null, - null, - new ArrayList<>(unprocessedManualAdd.getPositions()), - unprocessedManualAdd.getFileStatus(), - unprocessedManualAdd.getTextBefore(), - unprocessedManualAdd.getTextAfter(), - unprocessedManualAdd.getSourceId(), - new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries())); - - addRedactionPersistenceService.update(pendingDictAdd); - - // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it. - unprocessedManualAdd.setAddToDictionary(false); - unprocessedManualAdd.setAddToAllDossiers(false); - unprocessedManualAdd.setLegalBasis(""); - unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet()); - unprocessedManualAdd.setDictionaryEntryType(null); - - addRedactionPersistenceService.update(unprocessedManualAdd); - } - } - } - - - private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) { - - return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java deleted file mode 100644 index e2459e576..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ /dev/null @@ -1,396 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration; - -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; -import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; -import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; -import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; -import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob; -import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; -import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.redaction.v1.model.MigrationRequest; -import com.iqser.red.storage.commons.exception.StorageException; -import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; -import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; -import com.knecon.fforesight.tenantcommons.TenantContext; -import com.knecon.fforesight.tenantcommons.TenantProvider; -import com.knecon.fforesight.tenantcommons.model.TenantSyncEvent; -import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest; - -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class SaasMigrationService implements TenantSyncService { - - AutomaticAnalysisJob automaticAnalysisJob; - FileStatusPersistenceService fileStatusPersistenceService; - SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; - DossierService dossierService; - ManualRedactionProviderService manualRedactionProviderService; - TenantProvider tenantProvider; - IndexingService indexingService; - LayoutParsingRequestFactory layoutParsingRequestFactory; - RabbitTemplate rabbitTemplate; - FileManagementServiceSettings settings; - StorageService storageService; - SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; - UncompressedFilesMigrationService uncompressedFilesMigrationService; - ManualRedactionService manualRedactionService; - CommentService commentService; - RankDeDuplicationService rankDeDuplicationService; - SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService; - - - @Override - public synchronized void syncTenant(TenantSyncEvent tenantSyncEvent) { - - startMigrationForTenant(tenantSyncEvent.getTenantId()); - } - - // Persistence-Service needs to be scaled to 1. - - - public void startMigrationForTenant(String tenantId) { - - // TODO migrate rules. - automaticAnalysisJob.stopForTenant(tenantId); - - log.info("Starting uncompressed files migration ..."); - uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId); - log.info("Finished uncompressed files migration ..."); - - rankDeDuplicationService.deduplicate(); - int numberOfFiles = 0; - - var files = saasMigrationStatusPersistenceService.findAll(); - - for (var file : files) { - - var dossier = dossierService.getDossierById(file.getDossierId()); - - if (dossier.getHardDeletedTime() != null) { - if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) { - saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); - continue; - } else { - fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime()); - saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); - continue; - } - } - if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) { - saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); - continue; - } - - if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) { - log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus()); - continue; - } - - // delete NER_ENTITIES since offsets depend on old document structure. - storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES)); - - var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getDossierTemplate().getId(), file.getDossierId(), file.getFileId(), false); - - rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest); - - numberOfFiles++; - - } - - log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); - if (numberOfFiles == 0) { - finalizeMigration(); - } - } - - - public void startMigrationForFile(String dossierId, String fileId) { - - var dossier = dossierService.getDossierById(dossierId); - - if (dossier.getHardDeletedTime() != null) { - if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) { - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); - return; - } else { - fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime()); - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); - return; - } - } - if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) { - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); - return; - } - - log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId); - saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId); - var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getDossierTemplate().getId(), dossierId, fileId, false); - rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest); - } - - - public void handleLayoutParsingFinished(String dossierId, String fileId) { - - if (!layoutParsingFilesExist(dossierId, fileId)) { - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!"); - return; - } - - log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED); - - if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { - saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId); - } - - try { - indexingService.reindex(dossierId, Set.of(fileId), false); - - String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId(); - - rabbitTemplate.convertAndSend(MIGRATION_REQUEST_QUEUE, - MigrationRequest.builder() - .dossierTemplateId(dossierTemplateId) - .dossierId(dossierId) - .fileId(fileId) - .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) - .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) - .entitiesWithComments(commentService.getCommentCounts(fileId).keySet()) - .build()); - } catch (Exception e) { - log.error("Queuing of entityLog migration failed with {}", e.getMessage()); - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage())); - } - - } - - - private boolean layoutParsingFilesExist(String dossierId, String fileId) { - - return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); - } - - - public void handleEntityLogMigrationFinished(String dossierId, String fileId) { - - if (!entityLogMigrationFilesExist(dossierId, fileId)) { - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!"); - return; - } - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED); - - log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); - migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId); - } - - - private boolean entityLogMigrationFilesExist(String dossierId, String fileId) { - - return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists( - TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS)); - } - - - public void handleError(String dossierId, String fileId, String errorCause, String retryExchange) { - - var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId); - Integer numErrors = migrationEntry.getProcessingErrorCounter(); - if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) { - saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause); - rabbitTemplate.convertAndSend(retryExchange, TenantContext.getTenantId(), MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build()); - log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause); - } else { - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause); - log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause); - } - } - - - public void requeueErrorFiles() { - - automaticAnalysisJob.stopForTenant(TenantContext.getTenantId()); - saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR) - .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId())); - - } - - - private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) { - - MigratedIds migratedIds = getMigratedIds(dossierId, fileId); - Map oldToNewMapping = migratedIds.buildOldToNewMapping(); - updateAnnotationIds(dossierId, fileId, oldToNewMapping); - List forceRedactionIdsToDelete = migratedIds.getForceRedactionIdsToDelete(); - softDeleteForceRedactions(fileId, forceRedactionIdsToDelete); - log.info("Soft-deleted force redactions."); - List manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd(); - int count = addManualRedactionEntries(manualRedactionEntriesToAdd); - log.info("Added {} additional manual entries.", count); - deleteSectionGridAndNerEntitiesFiles(dossierId, fileId); - saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); - - log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); - finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change - } - - - private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) { - - try { - storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)); - } catch (StorageObjectDoesNotExist e) { - log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId); - } - - try { - storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); - } catch (StorageObjectDoesNotExist e) { - log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId); - } - } - - - private void softDeleteForceRedactions(String fileId, List forceRedactionIdsToDelete) { - - manualRedactionService.softDeleteForceRedactions(fileId, forceRedactionIdsToDelete); - } - - - private int addManualRedactionEntries(List manualRedactionEntriesToAdd) { - - manualRedactionEntriesToAdd.forEach(add -> { - if (add.getSection() != null && add.getSection().length() > 254) { - add.setSection(add.getSection().substring(0, 254)); - } - }); - - return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true); - } - - - public void revertMigrationForFile(String dossierId, String fileId) { - - log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId); - MigratedIds migratedIds = getMigratedIds(dossierId, fileId); - Map newToOldMapping = migratedIds.buildNewToOldMapping(); - updateAnnotationIds(dossierId, fileId, newToOldMapping); - deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd()); - undeleteForceRedactions(fileId, migratedIds.getForceRedactionIdsToDelete()); - saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId); - } - - - private void undeleteForceRedactions(String fileId, List forceRedactionIdsToDelete) { - - manualRedactionService.undeleteForceRedactions(fileId, forceRedactionIdsToDelete); - - } - - - private void deleteManualRedactionEntries(List manualRedactionEntriesToAdd) { - - manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd); - } - - - private void updateAnnotationIds(String dossierId, String fileId, Map idMapping) { - - try { - updateAnnotationIds(fileId, idMapping); - } catch (Exception e) { - String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s", - TenantContext.getTenantId(), - dossierId, - fileId, - e.getMessage()); - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message); - log.error(message); - throw e; - } - } - - - private void finalizeMigration() { - - if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) { -// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change - tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build()); - log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId()); - } - } - - - public void updateAnnotationIds(String fileId, Map idMapping) { - - AtomicInteger numUpdates = new AtomicInteger(0); - AtomicInteger numCommentUpdates = new AtomicInteger(0); - idMapping.forEach((key, value) -> { - AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key); - AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value)); - }); - log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId); - } - - - private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) { - - return new AnnotationEntityId(annotationId, fileId); - } - - - private MigratedIds getMigratedIds(String dossierId, String fileId) { - - try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS), MigratedIds.class); - } catch (StorageObjectDoesNotExist e) { - throw new NotFoundException(String.format("MigratedIds does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId)); - } catch (StorageException e) { - throw new InternalServerErrorException(e.getMessage()); - } - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index dcdcca15b..8aa2cbee8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -14,7 +14,6 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; @@ -38,7 +37,6 @@ public class AutomaticAnalysisJob implements Job { private final FileStatusService fileStatusService; private final TenantProvider tenantProvider; private final ObservationRegistry observationRegistry; - private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; @Setter private boolean schedulingStopped; @@ -69,11 +67,6 @@ public class AutomaticAnalysisJob implements Job { TenantContext.setTenantId(tenant.getTenantId()); - if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) { - log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId()); - return; - } - String queueName = MessagingConfiguration.REDACTION_REQUEST_QUEUE_PREFIX + "_" + tenant.getTenantId(); var redactionQueueInfo = amqpAdmin.getQueueInfo(queueName); if (redactionQueueInfo != null) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java deleted file mode 100644 index a2a71292e..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; - -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; - -import jakarta.transaction.Transactional; -import lombok.RequiredArgsConstructor; - -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -@RequiredArgsConstructor -public class SaasMigrationStatusPersistenceService { - - private final SaasMigrationStatusRepository saasMigrationStatusRepository; - - - public List findAllByStatus(SaasMigrationStatus status) { - - return saasMigrationStatusRepository.findAllByStatus(status); - } - - - public SaasMigrationStatusEntity findById(String fileId) { - - var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId); - if (migrationStatusOptional.isPresent()) { - return migrationStatusOptional.get(); - } - throw new NotFoundException("No migration entry found for fileId" + fileId); - } - - - public boolean isMigrating(String fileId) { - - var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId); - return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED; - } - - public boolean migrationFinishedForTenant() { - - return saasMigrationStatusRepository.findAllWhereStatusNotFinishedAndNotError() == 0; - } - - - @Transactional - public void createMigrationRequiredStatus(String dossierId, String fileId) { - - saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build()); - } - - - @Transactional - public void updateStatus(String fileId, SaasMigrationStatus status) { - - saasMigrationStatusRepository.updateStatus(fileId, status); - } - - - @Transactional - public void updateErrorStatus(String fileId, String errorCause) { - - saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause); - } - - - @Transactional - public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) { - - saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause); - } - - - public int countByStatus(SaasMigrationStatus status) { - - return saasMigrationStatusRepository.countByStatus(status); - } - - - public int countAll() { - - return saasMigrationStatusRepository.countAll(); - } - - - public List findAll() { - - return saasMigrationStatusRepository.findAll(); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java deleted file mode 100644 index e6ced6cea..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; - -import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; - -import java.util.List; - -public interface SaasMigrationStatusRepository extends JpaRepository { - - List findAllByStatus(SaasMigrationStatus status); - - - @Modifying - @Query("update SaasMigrationStatusEntity e set e.status = :status where e.fileId = :fileId") - void updateStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status); - - - @Modifying - @Query("update SaasMigrationStatusEntity e set e.status = :status, e.errorCause = :errorCause where e.fileId = :fileId") - void updateErrorStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status, @Param("errorCause") String errorCause); - - - @Modifying - @Query("update SaasMigrationStatusEntity e set e.processingErrorCounter = :processingErrorCounter, e.errorCause = :errorCause where e.fileId = :fileId") - void updateErrorCounter(@Param("fileId") String fileId, @Param("processingErrorCounter") Integer processingErrorCounter, @Param("errorCause") String errorCause); - - - @Query("select count(*) from SaasMigrationStatusEntity e where e.status = :status") - int countByStatus(@Param("status") SaasMigrationStatus status); - - - @Query("select count(*) from SaasMigrationStatusEntity") - int countAll(); - - - @Query("select count(*) from SaasMigrationStatusEntity e where e.status != 'FINISHED' and e.status != 'ERROR'") - int findAllWhereStatusNotFinishedAndNotError(); - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index a347033e4..67dc86d9f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -11,14 +11,12 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; -import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -40,8 +38,6 @@ public class LayoutParsingFinishedMessageReceiver { private final FileStatusService fileStatusService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObjectMapper objectMapper; - private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; - private final SaasMigrationService saasMigrationService; private final ImageSimilarityService imageSimilarityService; private final WebsocketService websocketService; @@ -53,11 +49,7 @@ public class LayoutParsingFinishedMessageReceiver { var dossierId = QueueMessageIdentifierService.parseDossierId(response.identifier()); var fileId = QueueMessageIdentifierService.parseFileId(response.identifier()); log.info("Layout parsing has finished for {}/{} in {}", dossierId, fileId, LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE); - if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(response.identifier()))) { - saasMigrationService.handleLayoutParsingFinished(QueueMessageIdentifierService.parseDossierId(response.identifier()), - QueueMessageIdentifierService.parseFileId(response.identifier())); - return; - } + fileStatusService.setStatusAnalyse(QueueMessageIdentifierService.parseDossierId(response.identifier()), QueueMessageIdentifierService.parseFileId(response.identifier()), @@ -88,13 +80,6 @@ public class LayoutParsingFinishedMessageReceiver { if (errorCause == null) { errorCause = "Error occured during layout parsing!"; } - if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()))) { - saasMigrationService.handleError(QueueMessageIdentifierService.parseDossierId(analyzeRequest.identifier()), - QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()), - errorCause, - LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE); - return; - } OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER); timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java deleted file mode 100644 index a098afcf9..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.queue; - -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_DLQ; -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE; -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_RESPONSE_QUEUE; -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER; - -import org.springframework.amqp.core.Message; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; -import com.iqser.red.service.redaction.v1.model.MigrationRequest; -import com.iqser.red.service.redaction.v1.model.MigrationResponse; - -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class RedactionServiceSaasMigrationMessageReceiver { - - private final SaasMigrationService saasMigrationService; - private final ObjectMapper objectMapper; - - - @SneakyThrows - @RabbitListener(queues = MIGRATION_RESPONSE_QUEUE) - public void receive(MigrationResponse response) { - - saasMigrationService.handleEntityLogMigrationFinished(response.getDossierId(), response.getFileId()); - - log.info("Received message {} in {}", response, MIGRATION_RESPONSE_QUEUE); - } - - - @SneakyThrows - @RabbitListener(queues = MIGRATION_DLQ) - public void handleDLQMessage(Message failedMessage) { - - var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class); - String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER); - if (errorCause == null) { - errorCause = "Error occured during entityLog migration!"; - } - saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_REQUEST_QUEUE); - - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151-drop-saas-migration-table.changelog.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151-drop-saas-migration-table.changelog.yaml new file mode 100644 index 000000000..e8c3729a4 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151-drop-saas-migration-table.changelog.yaml @@ -0,0 +1,12 @@ +databaseChangeLog: + - changeSet: + id: drop-saas_migration_status-if-exists + author: dom + comment: drop saas_migration_status if exists + preConditions: + - onFail: MARK_RAN + - tableExists: + tableName: saas_migration_status + changes: + - dropTable: + tableName: saas_migration_status \ No newline at end of file