Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7384: migration fixes #363

Merged
kilian.schuettler1 merged 1 commits from RED-7384 into master 2024-02-26 11:39:30 +01:00
Conversation 4 Commits 1 Files Changed 7 +126 -35
7 changed files with 126 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -10,11 +10,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Indexin
import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
@ -35,6 +37,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@ -62,6 +65,7 @@ public class SaasMigrationService implements TenantSyncService {
StorageService storageService;
SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
UncompressedFilesMigrationService uncompressedFilesMigrationService;
ManualRedactionService manualRedactionService;
@Override
@ -188,7 +192,7 @@ public class SaasMigrationService implements TenantSyncService {
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
migrateAnnotationIds(dossierId, fileId);
migrateAnnotationIdsAndAddManualAddRedactions(dossierId, fileId);
}
@ -215,11 +219,14 @@ public class SaasMigrationService implements TenantSyncService {
}
private void migrateAnnotationIds(String dossierId, String fileId) {
private void migrateAnnotationIdsAndAddManualAddRedactions(String dossierId, String fileId) {
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
updateAnnotationIds(dossierId, fileId, oldToNewMapping);
List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
log.info("Added {} additional manual entries.", count);
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
@ -227,16 +234,29 @@ public class SaasMigrationService implements TenantSyncService {
}
private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
}
public void revertMigrationForFile(String dossierId, String fileId) {
log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
updateAnnotationIds(dossierId, fileId, newToOldMapping);
deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
}
private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
}
private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
try {

74
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -15,8 +15,10 @@ import org.springframework.transaction.annotation.Transactional;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
@ -33,6 +35,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
@ -46,6 +49,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
@ -103,8 +107,7 @@ public class ManualRedactionService {
Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
@ -113,8 +116,7 @@ public class ManualRedactionService {
FileModel fileStatus = fileStatusService.getStatus(fileId);
if (!manualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities))
.build();
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities)).build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
} else {
reprocess(dossierId, fileId);
@ -158,8 +160,7 @@ public class ManualRedactionService {
removeRedactionPersistenceService.updateModifiedDictionaries(fileId, removeRedactionRequest.getAnnotationId(), removedFromDictionary, typeIdsOfModifiedDictionaries);
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
reprocess(dossierId, fileId);
@ -184,8 +185,7 @@ public class ManualRedactionService {
forceRedactionRequest.getComment(),
forceRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
reprocess(dossierId, fileId);
@ -210,8 +210,7 @@ public class ManualRedactionService {
legalBasisChangeRequest.getComment(),
legalBasisChangeRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
}
reprocess(dossierId, fileId);
fileStatusPersistenceService.setLastManualChangeDate(fileId, OffsetDateTime.now());
@ -250,8 +249,7 @@ public class ManualRedactionService {
recategorizationRequest.getComment(),
recategorizationRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build());
}
reprocess(dossierId, fileId);
@ -287,8 +285,7 @@ public class ManualRedactionService {
resizeRedactionRequest.getAnnotationId(),
resizeRedactionRequest.getComment(),
resizeRedactionRequest.getUser());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId)
.build());
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId,
@ -309,8 +306,7 @@ public class ManualRedactionService {
FileModel fileStatus = fileStatusService.getStatus(fileId);
if (!manualResizeRedactionEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities))
.build();
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities)).build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions);
} else {
reprocess(dossierId, fileId);
@ -419,4 +415,50 @@ public class ManualRedactionService {
}
}
public int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd, boolean addAsProcessed) {
List<ManualRedactionEntryEntity> manualRedactionEntryEntities = manualRedactionEntriesToAdd.stream()
.map(this::toManualRedactionEntryEntity)
.toList();
if (addAsProcessed) {
manualRedactionEntryEntities.forEach(e -> e.setProcessedDate(OffsetDateTime.now()));
}
return addRedactionPersistenceService.insert(manualRedactionEntryEntities).size();
}
private ManualRedactionEntryEntity toManualRedactionEntryEntity(ManualRedactionEntry manualRedactionEntry) {
var file = fileStatusPersistenceService.getStatus(manualRedactionEntry.getFileId());
String dossierId = file.getDossierId();
String dossierTemplateId = dossierPersistenceService.getDossierTemplateId(dossierId);
String type = manualRedactionEntry.getType();
ManualRedactionEntryEntity manualRedactionEntryEntity = MagicConverter.convert(manualRedactionEntry, ManualRedactionEntryEntity.class);
String typeId;
if (manualRedactionEntry.isAddToDossierDictionary()) {
typeId = TypeIdUtils.toTypeId(type, dossierTemplateId, dossierId);
} else {
typeId = TypeIdUtils.toTypeId(type, dossierTemplateId);
}
manualRedactionEntryEntity.setTypeId(typeId);
manualRedactionEntryEntity.setTypeIdsOfModifiedDictionaries(Set.of(typeId));
manualRedactionEntryEntity.setId(new AnnotationEntityId(manualRedactionEntry.getAnnotationId(), manualRedactionEntry.getFileId()));
manualRedactionEntryEntity.setPositions(manualRedactionEntry.getPositions()
.stream()
.map(p -> MagicConverter.convert(p, RectangleEntity.class))
.toList());
manualRedactionEntryEntity.setRequestDate(file.getAdded());
manualRedactionEntryEntity.setFileStatus(file);
return manualRedactionEntryEntity;
}
public void deleteManualRedactionEntries(List<ManualRedactionEntry> list) {
list.forEach(manualRedactionEntry -> addRedactionPersistenceService.hardDelete(manualRedactionEntry.getFileId(), manualRedactionEntry.getAnnotationId()));
}
}

46
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java
View File

@ -116,23 +116,28 @@ public class DossierPersistenceService {
@Transactional
public void update(String dossierId, CreateOrUpdateDossierRequest createOrUpdateDossierRequest) {
dossierRepository.findById(dossierId).ifPresent(dossier -> {
BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId");
dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId()));
dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
dossierRepository.findById(dossierId)
.ifPresent(dossier -> {
BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId");
dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId()));
dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
dossier.getReportTemplates()
.forEach(t -> t.getDossiers().removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId())));
var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds());
reportTemplates.forEach(r -> {
if (!r.getDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()).contains(dossierId)) {
r.getDossiers().add(dossier);
}
});
dossier.setReportTemplates(reportTemplates);
this.handleDossierStatus(createOrUpdateDossierRequest, dossier);
this.handleWatermark(createOrUpdateDossierRequest, dossier);
});
dossier.getReportTemplates()
.forEach(t -> t.getDossiers()
.removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId())));
var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds());
reportTemplates.forEach(r -> {
if (!r.getDossiers()
.stream()
.map(DossierEntity::getId)
.collect(Collectors.toSet()).contains(dossierId)) {
r.getDossiers().add(dossier);
}
});
dossier.setReportTemplates(reportTemplates);
this.handleDossierStatus(createOrUpdateDossierRequest, dossier);
this.handleWatermark(createOrUpdateDossierRequest, dossier);
});
}
@ -151,7 +156,8 @@ public class DossierPersistenceService {
public DossierEntity findByDossierId(String dossierId) {
return dossierRepository.findById(dossierId).orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
return dossierRepository.findById(dossierId)
.orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
}
@ -250,4 +256,10 @@ public class DossierPersistenceService {
}
}
public String getDossierTemplateId(String dossierId) {
return dossierRepository.findDossierTemplateId(dossierId);
}
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -47,6 +47,13 @@ public class AddRedactionPersistenceService {
}
@Transactional
public List<ManualRedactionEntryEntity> insert(List<ManualRedactionEntryEntity> manualRedactionEntryEntities) {
return manualRedactionRepository.saveAllAndFlush(manualRedactionEntryEntities);
}
public Optional<ManualRedactionEntryEntity> findById(String annotationId, String fileId) {
return manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId));
@ -110,6 +117,7 @@ public class AddRedactionPersistenceService {
manualRedactionRepository.updateSoftDelete(new AnnotationEntityId(annotationId, fileId), null);
}
@Transactional
public void update(ManualRedactionEntryEntity manualRedactionEntry) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java
View File

@ -94,4 +94,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
@Query("select count(d) from DossierEntity d where d.watermarkId = :watermarkId or d.previewWatermarkId = :watermarkId")
int countDossiersWithWatermarkInUse(@Param("watermarkId") long watermarkId);
@Query("select d.dossierTemplateId from DossierEntity d where d.id = :dossierId")
String findDossierTemplateId(@Param("dossierId") String dossierId);
}

4
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java
View File

@ -4,6 +4,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -17,6 +19,8 @@ public class MigratedIds {
List<Mapping> mappings;
List<ManualRedactionEntry> manualRedactionEntriesToAdd;
public Map<String, String> buildOldToNewMapping() {