diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 2f2535827..aacac11a3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -10,11 +10,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Indexin import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; import com.iqser.red.service.redaction.v1.model.MigrationRequest; @@ -35,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -62,6 +65,7 @@ public class SaasMigrationService implements TenantSyncService { StorageService storageService; SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; UncompressedFilesMigrationService uncompressedFilesMigrationService; + ManualRedactionService manualRedactionService; @Override @@ -188,7 +192,7 @@ public class SaasMigrationService implements TenantSyncService { saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED); log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); - migrateAnnotationIds(dossierId, fileId); + migrateAnnotationIdsAndAddManualAddRedactions(dossierId, fileId); } @@ -215,11 +219,14 @@ public class SaasMigrationService implements TenantSyncService { } - private void migrateAnnotationIds(String dossierId, String fileId) { + private void migrateAnnotationIdsAndAddManualAddRedactions(String dossierId, String fileId) { MigratedIds migratedIds = getMigratedIds(dossierId, fileId); Map oldToNewMapping = migratedIds.buildOldToNewMapping(); updateAnnotationIds(dossierId, fileId, oldToNewMapping); + List manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd(); + int count = addManualRedactionEntries(manualRedactionEntriesToAdd); + log.info("Added {} additional manual entries.", count); saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); @@ -227,16 +234,29 @@ public class SaasMigrationService implements TenantSyncService { } + private int addManualRedactionEntries(List manualRedactionEntriesToAdd) { + + return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true); + } + + public void revertMigrationForFile(String dossierId, String fileId) { log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId); MigratedIds migratedIds = getMigratedIds(dossierId, fileId); Map newToOldMapping = migratedIds.buildNewToOldMapping(); updateAnnotationIds(dossierId, fileId, newToOldMapping); + deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd()); saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId); } + private void deleteManualRedactionEntries(List manualRedactionEntriesToAdd) { + + manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd); + } + + private void updateAnnotationIds(String dossierId, String fileId, Map idMapping) { try { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java index 53e0850e5..195ad5151 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java @@ -15,8 +15,10 @@ import org.springframework.transaction.annotation.Transactional; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; @@ -33,6 +35,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionMapper; import com.iqser.red.service.persistence.management.v1.processor.utils.ManualResizeRedactionMapper; +import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest; @@ -46,6 +49,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; @@ -103,8 +107,7 @@ public class ManualRedactionService { Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } manualRedactionEntryEntities = manualRedactionEntryEntities.stream() @@ -113,8 +116,7 @@ public class ManualRedactionService { FileModel fileStatus = fileStatusService.getStatus(fileId); if (!manualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities)) - .build(); + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities)).build(); entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions); } else { reprocess(dossierId, fileId); @@ -158,8 +160,7 @@ public class ManualRedactionService { removeRedactionPersistenceService.updateModifiedDictionaries(fileId, removeRedactionRequest.getAnnotationId(), removedFromDictionary, typeIdsOfModifiedDictionaries); - response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); } reprocess(dossierId, fileId); @@ -184,8 +185,7 @@ public class ManualRedactionService { forceRedactionRequest.getComment(), forceRedactionRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); } reprocess(dossierId, fileId); @@ -210,8 +210,7 @@ public class ManualRedactionService { legalBasisChangeRequest.getComment(), legalBasisChangeRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build()); } reprocess(dossierId, fileId); fileStatusPersistenceService.setLastManualChangeDate(fileId, OffsetDateTime.now()); @@ -250,8 +249,7 @@ public class ManualRedactionService { recategorizationRequest.getComment(), recategorizationRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build()); } reprocess(dossierId, fileId); @@ -287,8 +285,7 @@ public class ManualRedactionService { resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); } Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId, @@ -309,8 +306,7 @@ public class ManualRedactionService { FileModel fileStatus = fileStatusService.getStatus(fileId); if (!manualResizeRedactionEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) { - ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities)) - .build(); + ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(convertResizeRedactions(manualResizeRedactionEntities)).build(); entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions); } else { reprocess(dossierId, fileId); @@ -419,4 +415,50 @@ public class ManualRedactionService { } } + + public int addManualRedactionEntries(List manualRedactionEntriesToAdd, boolean addAsProcessed) { + + List manualRedactionEntryEntities = manualRedactionEntriesToAdd.stream() + .map(this::toManualRedactionEntryEntity) + .toList(); + + if (addAsProcessed) { + manualRedactionEntryEntities.forEach(e -> e.setProcessedDate(OffsetDateTime.now())); + } + + return addRedactionPersistenceService.insert(manualRedactionEntryEntities).size(); + } + + + private ManualRedactionEntryEntity toManualRedactionEntryEntity(ManualRedactionEntry manualRedactionEntry) { + + var file = fileStatusPersistenceService.getStatus(manualRedactionEntry.getFileId()); + String dossierId = file.getDossierId(); + String dossierTemplateId = dossierPersistenceService.getDossierTemplateId(dossierId); + String type = manualRedactionEntry.getType(); + ManualRedactionEntryEntity manualRedactionEntryEntity = MagicConverter.convert(manualRedactionEntry, ManualRedactionEntryEntity.class); + String typeId; + if (manualRedactionEntry.isAddToDossierDictionary()) { + typeId = TypeIdUtils.toTypeId(type, dossierTemplateId, dossierId); + } else { + typeId = TypeIdUtils.toTypeId(type, dossierTemplateId); + } + manualRedactionEntryEntity.setTypeId(typeId); + manualRedactionEntryEntity.setTypeIdsOfModifiedDictionaries(Set.of(typeId)); + manualRedactionEntryEntity.setId(new AnnotationEntityId(manualRedactionEntry.getAnnotationId(), manualRedactionEntry.getFileId())); + manualRedactionEntryEntity.setPositions(manualRedactionEntry.getPositions() + .stream() + .map(p -> MagicConverter.convert(p, RectangleEntity.class)) + .toList()); + manualRedactionEntryEntity.setRequestDate(file.getAdded()); + manualRedactionEntryEntity.setFileStatus(file); + return manualRedactionEntryEntity; + } + + + public void deleteManualRedactionEntries(List list) { + + list.forEach(manualRedactionEntry -> addRedactionPersistenceService.hardDelete(manualRedactionEntry.getFileId(), manualRedactionEntry.getAnnotationId())); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 84ca37a67..3046c748c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -116,23 +116,28 @@ public class DossierPersistenceService { @Transactional public void update(String dossierId, CreateOrUpdateDossierRequest createOrUpdateDossierRequest) { - dossierRepository.findById(dossierId).ifPresent(dossier -> { - BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId"); - dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId())); - dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + dossierRepository.findById(dossierId) + .ifPresent(dossier -> { + BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId"); + dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId())); + dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - dossier.getReportTemplates() - .forEach(t -> t.getDossiers().removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId()))); - var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds()); - reportTemplates.forEach(r -> { - if (!r.getDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()).contains(dossierId)) { - r.getDossiers().add(dossier); - } - }); - dossier.setReportTemplates(reportTemplates); - this.handleDossierStatus(createOrUpdateDossierRequest, dossier); - this.handleWatermark(createOrUpdateDossierRequest, dossier); - }); + dossier.getReportTemplates() + .forEach(t -> t.getDossiers() + .removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId()))); + var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds()); + reportTemplates.forEach(r -> { + if (!r.getDossiers() + .stream() + .map(DossierEntity::getId) + .collect(Collectors.toSet()).contains(dossierId)) { + r.getDossiers().add(dossier); + } + }); + dossier.setReportTemplates(reportTemplates); + this.handleDossierStatus(createOrUpdateDossierRequest, dossier); + this.handleWatermark(createOrUpdateDossierRequest, dossier); + }); } @@ -151,7 +156,8 @@ public class DossierPersistenceService { public DossierEntity findByDossierId(String dossierId) { - return dossierRepository.findById(dossierId).orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE)); + return dossierRepository.findById(dossierId) + .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE)); } @@ -250,4 +256,10 @@ public class DossierPersistenceService { } } + + public String getDossierTemplateId(String dossierId) { + + return dossierRepository.findDossierTemplateId(dossierId); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java index b6bb7ee43..b38dd95c7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index 08526e56d..08aa43a4e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -47,6 +47,13 @@ public class AddRedactionPersistenceService { } + @Transactional + public List insert(List manualRedactionEntryEntities) { + + return manualRedactionRepository.saveAllAndFlush(manualRedactionEntryEntities); + } + + public Optional findById(String annotationId, String fileId) { return manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId)); @@ -110,6 +117,7 @@ public class AddRedactionPersistenceService { manualRedactionRepository.updateSoftDelete(new AnnotationEntityId(annotationId, fileId), null); } + @Transactional public void update(ManualRedactionEntryEntity manualRedactionEntry) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 2991a75cb..63fcbc696 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -94,4 +94,8 @@ public interface DossierRepository extends JpaRepository @Query("select count(d) from DossierEntity d where d.watermarkId = :watermarkId or d.previewWatermarkId = :watermarkId") int countDossiersWithWatermarkInUse(@Param("watermarkId") long watermarkId); + + @Query("select d.dossierTemplateId from DossierEntity d where d.id = :dossierId") + String findDossierTemplateId(@Param("dossierId") String dossierId); + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java index 09752f243..474c52fac 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -17,6 +19,8 @@ public class MigratedIds { List mappings; + List manualRedactionEntriesToAdd; + public Map buildOldToNewMapping() {