Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-10715: Remove Saas migration

Browse Source
This commit is contained in:
Dominique Eifländer 2025-01-13 11:49:06 +01:00
parent 205f9c678e
commit d7d46d5429
12 changed files with 13 additions and 1067 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java vendored
View File

@ -1,111 +0,0 @@
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
@RestController
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@RequiredArgsConstructor
public class MigrationStatusController implements MigrationStatusResource {
SaasMigrationService saasMigrationService;
SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
FileStatusService fileStatusService;
public MigrationStatusResponse migrationStatus() {
int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll();
Map<SaasMigrationStatus, Integer> filesInStatus = new HashMap<>();
filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED));
filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED));
filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED));
filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED));
filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED));
filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR));
var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
var errorCauses = filesInErrorState.stream()
.collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
}
@Override
public ResponseEntity<?> startMigrationForFile(String dossierId, String fileId) {
if (!fileStatusService.fileExists(fileId)) {
throw new NotFoundException(String.format("File with id %s does not exist", fileId));
}
saasMigrationService.startMigrationForFile(dossierId, fileId);
return ResponseEntity.ok().build();
}
@Override
public ResponseEntity<?> revertMigrationForFile(String dossierId, String fileId) {
if (!fileStatusService.fileExists(fileId)) {
throw new NotFoundException(String.format("File with id %s does not exist", fileId));
}
if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) {
throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId));
}
saasMigrationService.revertMigrationForFile(dossierId, fileId);
return ResponseEntity.ok().build();
}
@Override
public ResponseEntity<?> requeueErrorFiles() {
MigrationStatusResponse migrationStatus = migrationStatus();
if (!migrationIsFinished(migrationStatus)) {
throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!");
}
saasMigrationService.requeueErrorFiles();
return ResponseEntity.ok().build();
}
private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) {
return migrationStatus.getFilesInStatus().entrySet()
.stream()
.filter(e -> e.getValue() > 0)
.allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR));
}
}

56
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java vendored
View File

@ -1,56 +0,0 @@
package com.iqser.red.service.persistence.service.v1.api.external.resource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
public interface MigrationStatusResource {
String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status";
String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration";
String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration";
String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration";
String FILE_ID = "fileId";
String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
String DOSSIER_ID = "dossierId";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
@ResponseBody
@PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Show the status of the migration", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
MigrationStatusResponse migrationStatus();
@ResponseBody
@PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
@Operation(summary = "Start SAAS migration for specific file", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
ResponseEntity<?> startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
@ResponseBody
@PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
@Operation(summary = "Start SAAS migration for specific file", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
ResponseEntity<?> revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
@ResponseBody
@PostMapping(value = RETRY_MIGRATION_REST_PATH)
@Operation(summary = "Restart SAAS migration for all files in error state", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
ResponseEntity<?> requeueErrorFiles();
}

35
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java
View File

@ -1,35 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.migration;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Entity
@Table(name = "saas_migration_status")
public class SaasMigrationStatusEntity {
@Id
private String fileId;
@Column
private String dossierId;
@Column
@Enumerated(EnumType.STRING)
private SaasMigrationStatus status;
@Column
private Integer processingErrorCounter;
@Column
private String errorCause;
}

170
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java
View File

@ -1,170 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.migration;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import jakarta.transaction.Transactional;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@Transactional
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class SaasAnnotationIdMigrationService {
ManualRedactionRepository manualRedactionRepository;
RemoveRedactionRepository removeRedactionRepository;
ForceRedactionRepository forceRedactionRepository;
ResizeRedactionRepository resizeRedactionRepository;
RecategorizationRepository recategorizationRepository;
LegalBasisChangeRepository legalBasisChangeRepository;
CommentRepository commentRepository;
FileRepository fileRepository;
public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), ManualRedactionEntryEntity.class);
newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
newEntry.setId(newAnnotationEntityId);
manualRedactionRepository.deleteById(oldAnnotationEntityId);
manualRedactionRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), IdRemovalEntity.class);
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
newEntry.setId(newAnnotationEntityId);
removeRedactionRepository.deleteById(oldAnnotationEntityId);
removeRedactionRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), ManualForceRedactionEntity.class);
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
newEntry.setId(newAnnotationEntityId);
forceRedactionRepository.deleteById(oldAnnotationEntityId);
forceRedactionRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), ManualResizeRedactionEntity.class);
newEntry.setId(newAnnotationEntityId);
newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
resizeRedactionRepository.deleteById(oldAnnotationEntityId);
resizeRedactionRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), ManualRecategorizationEntity.class);
newEntry.setId(newAnnotationEntityId);
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
recategorizationRepository.deleteById(oldAnnotationEntityId);
recategorizationRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
return 0;
}
var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
if (oldEntry.isPresent()) {
var newEntry = MagicConverter.convert(oldEntry.get(), ManualLegalBasisChangeEntity.class);
newEntry.setId(newAnnotationEntityId);
newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
.get());
legalBasisChangeRepository.deleteById(oldAnnotationEntityId);
legalBasisChangeRepository.save(newEntry);
return 1;
}
return 0;
}
public int updateCommentIds(String fileId, String key, String value) {
if (key.equals(value)) {
return 0;
}
return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
}
}

83
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java
View File

@ -1,83 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.migration;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class SaasMigrationManualChangesUpdateService {
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final HashFunction hashFunction = Hashing.murmur3_128();
public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) {
var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
for (var unprocessedManualAdd : unprocessedManualAdds) {
if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) {
continue;
}
if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
// copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
// It has no functionality, its only there, such that the UI can show a pending change.
ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId),
unprocessedManualAdd.getUser(),
unprocessedManualAdd.getTypeId(),
unprocessedManualAdd.getValue(),
unprocessedManualAdd.getReason(),
unprocessedManualAdd.getLegalBasis(),
unprocessedManualAdd.getSection(),
unprocessedManualAdd.isRectangle(),
unprocessedManualAdd.isAddToDictionary(),
unprocessedManualAdd.isAddToAllDossiers(),
unprocessedManualAdd.isAddToDossierDictionary(),
DictionaryEntryType.ENTRY,
unprocessedManualAdd.getRequestDate(),
null,
null,
new ArrayList<>(unprocessedManualAdd.getPositions()),
unprocessedManualAdd.getFileStatus(),
unprocessedManualAdd.getTextBefore(),
unprocessedManualAdd.getTextAfter(),
unprocessedManualAdd.getSourceId(),
new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()));
addRedactionPersistenceService.update(pendingDictAdd);
// change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it.
unprocessedManualAdd.setAddToDictionary(false);
unprocessedManualAdd.setAddToAllDossiers(false);
unprocessedManualAdd.setLegalBasis("");
unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
unprocessedManualAdd.setDictionaryEntryType(null);
addRedactionPersistenceService.update(unprocessedManualAdd);
}
}
}
private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) {
return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId);
}
}

396
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -1,396 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.migration;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantSyncEvent;
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class SaasMigrationService implements TenantSyncService {
AutomaticAnalysisJob automaticAnalysisJob;
FileStatusPersistenceService fileStatusPersistenceService;
SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
DossierService dossierService;
ManualRedactionProviderService manualRedactionProviderService;
TenantProvider tenantProvider;
IndexingService indexingService;
LayoutParsingRequestFactory layoutParsingRequestFactory;
RabbitTemplate rabbitTemplate;
FileManagementServiceSettings settings;
StorageService storageService;
SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
UncompressedFilesMigrationService uncompressedFilesMigrationService;
ManualRedactionService manualRedactionService;
CommentService commentService;
RankDeDuplicationService rankDeDuplicationService;
SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService;
@Override
public synchronized void syncTenant(TenantSyncEvent tenantSyncEvent) {
startMigrationForTenant(tenantSyncEvent.getTenantId());
}
// Persistence-Service needs to be scaled to 1.
public void startMigrationForTenant(String tenantId) {
// TODO migrate rules.
automaticAnalysisJob.stopForTenant(tenantId);
log.info("Starting uncompressed files migration ...");
uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
log.info("Finished uncompressed files migration ...");
rankDeDuplicationService.deduplicate();
int numberOfFiles = 0;
var files = saasMigrationStatusPersistenceService.findAll();
for (var file : files) {
var dossier = dossierService.getDossierById(file.getDossierId());
if (dossier.getHardDeletedTime() != null) {
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
} else {
fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime());
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
}
}
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
}
if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
continue;
}
// delete NER_ENTITIES since offsets depend on old document structure.
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES));
var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getDossierTemplate().getId(), file.getDossierId(), file.getFileId(), false);
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
numberOfFiles++;
}
log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
if (numberOfFiles == 0) {
finalizeMigration();
}
}
public void startMigrationForFile(String dossierId, String fileId) {
var dossier = dossierService.getDossierById(dossierId);
if (dossier.getHardDeletedTime() != null) {
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
} else {
fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime());
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
}
}
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
}
log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getDossierTemplate().getId(), dossierId, fileId, false);
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
}
public void handleLayoutParsingFinished(String dossierId, String fileId) {
if (!layoutParsingFilesExist(dossierId, fileId)) {
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!");
return;
}
log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED);
if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId);
}
try {
indexingService.reindex(dossierId, Set.of(fileId), false);
String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
rabbitTemplate.convertAndSend(MIGRATION_REQUEST_QUEUE,
MigrationRequest.builder()
.dossierTemplateId(dossierTemplateId)
.dossierId(dossierId)
.fileId(fileId)
.fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
.entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
.build());
} catch (Exception e) {
log.error("Queuing of entityLog migration failed with {}", e.getMessage());
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
}
}
private boolean layoutParsingFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
}
public void handleEntityLogMigrationFinished(String dossierId, String fileId) {
if (!entityLogMigrationFilesExist(dossierId, fileId)) {
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!");
return;
}
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId);
}
private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
}
public void handleError(String dossierId, String fileId, String errorCause, String retryExchange) {
var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId);
Integer numErrors = migrationEntry.getProcessingErrorCounter();
if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) {
saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause);
rabbitTemplate.convertAndSend(retryExchange, TenantContext.getTenantId(), MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build());
log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
} else {
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause);
log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
}
}
public void requeueErrorFiles() {
automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
.forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
}
private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
updateAnnotationIds(dossierId, fileId, oldToNewMapping);
List<String> forceRedactionIdsToDelete = migratedIds.getForceRedactionIdsToDelete();
softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
log.info("Soft-deleted force redactions.");
List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
log.info("Added {} additional manual entries.", count);
deleteSectionGridAndNerEntitiesFiles(dossierId, fileId);
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
}
private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) {
try {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
} catch (StorageObjectDoesNotExist e) {
log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId);
}
try {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
} catch (StorageObjectDoesNotExist e) {
log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId);
}
}
private void softDeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
manualRedactionService.softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
}
private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
manualRedactionEntriesToAdd.forEach(add -> {
if (add.getSection() != null && add.getSection().length() > 254) {
add.setSection(add.getSection().substring(0, 254));
}
});
return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
}
public void revertMigrationForFile(String dossierId, String fileId) {
log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
updateAnnotationIds(dossierId, fileId, newToOldMapping);
deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
undeleteForceRedactions(fileId, migratedIds.getForceRedactionIdsToDelete());
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
}
private void undeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
manualRedactionService.undeleteForceRedactions(fileId, forceRedactionIdsToDelete);
}
private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
}
private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
try {
updateAnnotationIds(fileId, idMapping);
} catch (Exception e) {
String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
TenantContext.getTenantId(),
dossierId,
fileId,
e.getMessage());
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
log.error(message);
throw e;
}
}
private void finalizeMigration() {
if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
}
}
public void updateAnnotationIds(String fileId, Map<String, String> idMapping) {
AtomicInteger numUpdates = new AtomicInteger(0);
AtomicInteger numCommentUpdates = new AtomicInteger(0);
idMapping.forEach((key, value) -> {
AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
});
log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
}
private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
return new AnnotationEntityId(annotationId, fileId);
}
private MigratedIds getMigratedIds(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS), MigratedIds.class);
} catch (StorageObjectDoesNotExist e) {
throw new NotFoundException(String.format("MigratedIds does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
} catch (StorageException e) {
throw new InternalServerErrorException(e.getMessage());
}
}
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java
View File

@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
@ -38,7 +37,6 @@ public class AutomaticAnalysisJob implements Job {
private final FileStatusService fileStatusService;
private final TenantProvider tenantProvider;
private final ObservationRegistry observationRegistry;
private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
@Setter
private boolean schedulingStopped;
@ -69,11 +67,6 @@ public class AutomaticAnalysisJob implements Job {
TenantContext.setTenantId(tenant.getTenantId());
if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
return;
}
String queueName = MessagingConfiguration.REDACTION_REQUEST_QUEUE_PREFIX + "_" + tenant.getTenantId();
var redactionQueueInfo = amqpAdmin.getQueueInfo(queueName);
if (redactionQueueInfo != null) {

96
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java
View File

@ -1,96 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
@RequiredArgsConstructor
public class SaasMigrationStatusPersistenceService {
private final SaasMigrationStatusRepository saasMigrationStatusRepository;
public List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status) {
return saasMigrationStatusRepository.findAllByStatus(status);
}
public SaasMigrationStatusEntity findById(String fileId) {
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
if (migrationStatusOptional.isPresent()) {
return migrationStatusOptional.get();
}
throw new NotFoundException("No migration entry found for fileId" + fileId);
}
public boolean isMigrating(String fileId) {
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED;
}
public boolean migrationFinishedForTenant() {
return saasMigrationStatusRepository.findAllWhereStatusNotFinishedAndNotError() == 0;
}
@Transactional
public void createMigrationRequiredStatus(String dossierId, String fileId) {
saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build());
}
@Transactional
public void updateStatus(String fileId, SaasMigrationStatus status) {
saasMigrationStatusRepository.updateStatus(fileId, status);
}
@Transactional
public void updateErrorStatus(String fileId, String errorCause) {
saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause);
}
@Transactional
public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) {
saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause);
}
public int countByStatus(SaasMigrationStatus status) {
return saasMigrationStatusRepository.countByStatus(status);
}
public int countAll() {
return saasMigrationStatusRepository.countAll();
}
public List<SaasMigrationStatusEntity> findAll() {
return saasMigrationStatusRepository.findAll();
}
}

44
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java
View File

@ -1,44 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.util.List;
public interface SaasMigrationStatusRepository extends JpaRepository<SaasMigrationStatusEntity, String> {
List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status);
@Modifying
@Query("update SaasMigrationStatusEntity e set e.status = :status where e.fileId = :fileId")
void updateStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status);
@Modifying
@Query("update SaasMigrationStatusEntity e set e.status = :status, e.errorCause = :errorCause where e.fileId = :fileId")
void updateErrorStatus(@Param("fileId") String fileId, @Param("status") SaasMigrationStatus status, @Param("errorCause") String errorCause);
@Modifying
@Query("update SaasMigrationStatusEntity e set e.processingErrorCounter = :processingErrorCounter, e.errorCause = :errorCause where e.fileId = :fileId")
void updateErrorCounter(@Param("fileId") String fileId, @Param("processingErrorCounter") Integer processingErrorCounter, @Param("errorCause") String errorCause);
@Query("select count(*) from SaasMigrationStatusEntity e where e.status = :status")
int countByStatus(@Param("status") SaasMigrationStatus status);
@Query("select count(*) from SaasMigrationStatusEntity")
int countAll();
@Query("select count(*) from SaasMigrationStatusEntity e where e.status != 'FINISHED' and e.status != 'ERROR'")
int findAllWhereStatusNotFinishedAndNotError();
}

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java
View File

@ -11,14 +11,12 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService;
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -40,8 +38,6 @@ public class LayoutParsingFinishedMessageReceiver {
private final FileStatusService fileStatusService;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObjectMapper objectMapper;
private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
private final SaasMigrationService saasMigrationService;
private final ImageSimilarityService imageSimilarityService;
private final WebsocketService websocketService;
@ -53,11 +49,7 @@ public class LayoutParsingFinishedMessageReceiver {
var dossierId = QueueMessageIdentifierService.parseDossierId(response.identifier());
var fileId = QueueMessageIdentifierService.parseFileId(response.identifier());
log.info("Layout parsing has finished for {}/{} in {}", dossierId, fileId, LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE);
if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(response.identifier()))) {
saasMigrationService.handleLayoutParsingFinished(QueueMessageIdentifierService.parseDossierId(response.identifier()),
QueueMessageIdentifierService.parseFileId(response.identifier()));
return;
}
fileStatusService.setStatusAnalyse(QueueMessageIdentifierService.parseDossierId(response.identifier()),
QueueMessageIdentifierService.parseFileId(response.identifier()),
@ -88,13 +80,6 @@ public class LayoutParsingFinishedMessageReceiver {
if (errorCause == null) {
errorCause = "Error occured during layout parsing!";
}
if (saasMigrationStatusPersistenceService.isMigrating(QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()))) {
saasMigrationService.handleError(QueueMessageIdentifierService.parseDossierId(analyzeRequest.identifier()),
QueueMessageIdentifierService.parseFileId(analyzeRequest.identifier()),
errorCause,
LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
return;
}
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);

53
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java
View File

@ -1,53 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_DLQ;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_RESPONSE_QUEUE;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
import com.iqser.red.service.redaction.v1.model.MigrationResponse;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class RedactionServiceSaasMigrationMessageReceiver {
private final SaasMigrationService saasMigrationService;
private final ObjectMapper objectMapper;
@SneakyThrows
@RabbitListener(queues = MIGRATION_RESPONSE_QUEUE)
public void receive(MigrationResponse response) {
saasMigrationService.handleEntityLogMigrationFinished(response.getDossierId(), response.getFileId());
log.info("Received message {} in {}", response, MIGRATION_RESPONSE_QUEUE);
}
@SneakyThrows
@RabbitListener(queues = MIGRATION_DLQ)
public void handleDLQMessage(Message failedMessage) {
var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class);
String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
if (errorCause == null) {
errorCause = "Error occured during entityLog migration!";
}
saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_REQUEST_QUEUE);
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/151-drop-saas-migration-table.changelog.yaml Normal file
View File

@ -0,0 +1,12 @@
databaseChangeLog:
- changeSet:
id: drop-saas_migration_status-if-exists
author: dom
comment: drop saas_migration_status if exists
preConditions:
- onFail: MARK_RAN
- tableExists:
tableName: saas_migration_status
changes:
- dropTable:
tableName: saas_migration_status