RED-7834: fix migration issues
This commit is contained in:
Kilian Schüttler
Dominique Eifländer
parent
9b972984cf
commit
1ea65a256c
@ -5,49 +5,34 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
|
||||
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@Transactional
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class SaasAnnotationIdMigrationService {
|
||||
|
||||
private final ManualRedactionRepository manualRedactionRepository;
|
||||
private final RemoveRedactionRepository removeRedactionRepository;
|
||||
private final ForceRedactionRepository forceRedactionRepository;
|
||||
private final ResizeRedactionRepository resizeRedactionRepository;
|
||||
private final RecategorizationRepository recategorizationRepository;
|
||||
private final LegalBasisChangeRepository legalBasisChangeRepository;
|
||||
private final CommentRepository commentRepository;
|
||||
private final FileRepository fileRepository;
|
||||
ManualRedactionRepository manualRedactionRepository;
|
||||
RemoveRedactionRepository removeRedactionRepository;
|
||||
ForceRedactionRepository forceRedactionRepository;
|
||||
ResizeRedactionRepository resizeRedactionRepository;
|
||||
RecategorizationRepository recategorizationRepository;
|
||||
LegalBasisChangeRepository legalBasisChangeRepository;
|
||||
CommentRepository commentRepository;
|
||||
FileRepository fileRepository;
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateAnnotationIds(String fileId, Map<String, String> oldToNewMapping) {
|
||||
public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
AtomicInteger numUpdates = new AtomicInteger(0);
|
||||
AtomicInteger numCommentUpdates = new AtomicInteger(0);
|
||||
oldToNewMapping.forEach((key, value) -> {
|
||||
AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
|
||||
AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
|
||||
numUpdates.getAndAdd(updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numCommentUpdates.getAndAdd(commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value));
|
||||
});
|
||||
log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
|
||||
}
|
||||
|
||||
private int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -64,7 +49,8 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -80,7 +66,8 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -96,7 +83,8 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -113,7 +101,8 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -129,7 +118,8 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
|
||||
|
||||
var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
|
||||
if (oldEntry.isPresent()) {
|
||||
|
||||
@ -145,8 +135,9 @@ public class SaasAnnotationIdMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
|
||||
return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
|
||||
public int updateCommentIds(String fileId, String key, String value) {
|
||||
|
||||
return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
|
||||
@ -31,6 +33,8 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
|
||||
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
|
||||
@ -79,15 +83,32 @@ public class SaasMigrationService implements TenantSyncService {
|
||||
var dossiers = dossierService.getAllDossiers().stream().filter(dossier -> dossier.getHardDeletedTime() == null).toList();
|
||||
for (var dossier : dossiers) {
|
||||
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()).stream().filter(file -> file.getHardDeletedTime() == null).toList();
|
||||
var migrationStati = saasMigrationStatusPersistenceService.findAll()
|
||||
.stream()
|
||||
.collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus));
|
||||
for (var file : files) {
|
||||
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId());
|
||||
var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false);
|
||||
rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
|
||||
numberOfFiles++;
|
||||
if (notExistsOrError(file, migrationStati)) {
|
||||
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId());
|
||||
var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false);
|
||||
rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
|
||||
numberOfFiles++;
|
||||
} else {
|
||||
log.info("Skipping file with id {} and dossier {}, since it is already migrated or is currently being migrated!", file.getId(), file.getDossierId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
|
||||
|
||||
if (numberOfFiles == 0) {
|
||||
finalizeMigration();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean notExistsOrError(FileEntity file, Map<String, SaasMigrationStatus> migrationStati) {
|
||||
|
||||
return migrationStati.getOrDefault(file.getId(), SaasMigrationStatus.ERROR).equals(SaasMigrationStatus.ERROR);
|
||||
}
|
||||
|
||||
|
||||
@ -130,7 +151,6 @@ public class SaasMigrationService implements TenantSyncService {
|
||||
} catch (Exception e) {
|
||||
log.error("Queuing of entityLog migration failed with {}", e.getMessage());
|
||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -187,15 +207,26 @@ public class SaasMigrationService implements TenantSyncService {
|
||||
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
|
||||
Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
|
||||
try {
|
||||
saasAnnotationIdMigrationService.updateAnnotationIds(fileId, oldToNewMapping);
|
||||
updateAnnotationIds(fileId, oldToNewMapping);
|
||||
} catch (Exception e) {
|
||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, e.getMessage());
|
||||
log.error("Error during annotation id migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, e.getMessage());
|
||||
String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
|
||||
TenantContext.getTenantId(),
|
||||
dossierId,
|
||||
fileId,
|
||||
e.getMessage());
|
||||
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
|
||||
log.error(message);
|
||||
throw e;
|
||||
}
|
||||
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
|
||||
|
||||
log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
|
||||
finalizeMigration();
|
||||
}
|
||||
|
||||
|
||||
private void finalizeMigration() {
|
||||
|
||||
if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
|
||||
automaticAnalysisJob.startForTenant(TenantContext.getTenantId());
|
||||
log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
|
||||
@ -203,6 +234,31 @@ public class SaasMigrationService implements TenantSyncService {
|
||||
}
|
||||
|
||||
|
||||
public void updateAnnotationIds(String fileId, Map<String, String> oldToNewMapping) {
|
||||
|
||||
AtomicInteger numUpdates = new AtomicInteger(0);
|
||||
AtomicInteger numCommentUpdates = new AtomicInteger(0);
|
||||
oldToNewMapping.forEach((key, value) -> {
|
||||
AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
|
||||
AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
|
||||
numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
|
||||
});
|
||||
log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
|
||||
}
|
||||
|
||||
|
||||
private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
|
||||
|
||||
return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
|
||||
}
|
||||
|
||||
|
||||
private MigratedIds getMigratedIds(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
|
||||
@ -4,23 +4,27 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.migratio
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
|
||||
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SaasMigrationStatusPersistenceService {
|
||||
|
||||
private final SaasMigrationStatusRepository saasMigrationStatusRepository;
|
||||
|
||||
|
||||
public List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status) {
|
||||
|
||||
return saasMigrationStatusRepository.findAllByStatus(status);
|
||||
}
|
||||
|
||||
|
||||
public SaasMigrationStatusEntity findById(String fileId) {
|
||||
|
||||
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
|
||||
@ -30,40 +34,57 @@ public class SaasMigrationStatusPersistenceService {
|
||||
throw new NotFoundException("No migration entry found for fileId" + fileId);
|
||||
}
|
||||
|
||||
|
||||
public boolean isMigrating(String fileId) {
|
||||
|
||||
var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
|
||||
return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED;
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void createMigrationRequiredStatus(String dossierId, String fileId) {
|
||||
saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder()
|
||||
.fileId(fileId)
|
||||
.dossierId(dossierId)
|
||||
.status(SaasMigrationStatus.MIGRATION_REQUIRED)
|
||||
.build());
|
||||
|
||||
saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build());
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateStatus(String fileId, SaasMigrationStatus status) {
|
||||
|
||||
saasMigrationStatusRepository.updateStatus(fileId, status);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateErrorStatus(String fileId, String errorCause) {
|
||||
|
||||
saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause);
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) {
|
||||
|
||||
saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause);
|
||||
}
|
||||
|
||||
|
||||
public int countByStatus(SaasMigrationStatus status) {
|
||||
|
||||
return saasMigrationStatusRepository.countByStatus(status);
|
||||
}
|
||||
|
||||
|
||||
public int countAll() {
|
||||
|
||||
return saasMigrationStatusRepository.countAll();
|
||||
}
|
||||
|
||||
|
||||
public List<SaasMigrationStatusEntity> findAll() {
|
||||
|
||||
return saasMigrationStatusRepository.findAll();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -64,7 +64,9 @@ public class LayoutParsingFinishedMessageReceiver {
|
||||
LayoutParsingRequest analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class);
|
||||
log.info("Failed to process analyze request: {}", analyzeRequest);
|
||||
String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER);
|
||||
|
||||
if (errorCause == null) {
|
||||
errorCause = "Error occured during layout parsing!";
|
||||
}
|
||||
if (saasMigrationStatusPersistenceService.isMigrating(layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()))) {
|
||||
saasMigrationService.handleError(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()),
|
||||
layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()),
|
||||
|
||||
@ -42,6 +42,9 @@ public class RedactionServiceSaasMigrationMessageReceiver {
|
||||
|
||||
var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class);
|
||||
String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
|
||||
if (errorCause == null) {
|
||||
errorCause = "Error occured during entityLog migration!";
|
||||
}
|
||||
saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_QUEUE);
|
||||
|
||||
}
|
||||
|
||||
@ -9,7 +9,8 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/
|
||||
|
||||
logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
|
||||
|
||||
logging.type : ${LOGGING_TYPE:CONSOLE}
|
||||
|
||||
logging.type: ${LOGGING_TYPE:CONSOLE}
|
||||
kubernetes.namespace: ${NAMESPACE:default}
|
||||
project.version: 1.0-SNAPSHOT
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user