From 12cdc83ee3f71273f26fc665757c106f163ca4b1 Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Tue, 17 Oct 2023 14:52:10 +0300 Subject: [PATCH] RED-7763 - Use EntityLog in tests in persistence-service - remove unneeded migrations --- .../DeleteRemovedManualAddRedactions7.java | 73 --------- .../DictionaryToEntityMigration2.java | 89 ----------- .../migrations/EntityTypesMigration4.java | 74 --------- .../migrations/FileSizeMigration8.java | 66 -------- .../migration/migrations/IndexMigration1.java | 36 ----- .../ManualRedactionTypeMigration9.java | 84 ----------- .../migrations/MigrateHighlights3.java | 86 ----------- .../RemoveFalsePositiveManualRedactions6.java | 78 ---------- .../migrations/SimplifiedTextMigration12.java | 84 ----------- .../migrations/TypeToEntityMigration5.java | 142 ------------------ 10 files changed, 812 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DictionaryToEntityMigration2.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/EntityTypesMigration4.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/FileSizeMigration8.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/IndexMigration1.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeMigration9.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateHighlights3.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RemoveFalsePositiveManualRedactions6.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/SimplifiedTextMigration12.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/TypeToEntityMigration5.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java deleted file mode 100644 index 7ae908147..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class DeleteRemovedManualAddRedactions7 extends Migration { - - private static final String NAME = "Delete removed ManualAddRedactions"; - private static final long VERSION = 7; - - @Autowired - private AddRedactionPersistenceService addRedactionPersistenceService; - - @Autowired - private RemoveRedactionPersistenceService removeRedactionPersistenceService; - - @Autowired - private ManualRedactionProviderService manualRedactionProviderService; - - - public DeleteRemovedManualAddRedactions7() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - List addRedactions = addRedactionPersistenceService.findAllAddRedactions(); - - for (IdRemovalEntity idRemovalEntity : removeRedactionPersistenceService.findAllRemoveRedactions()) { - if (manualAddRedactionsContains(addRedactions, idRemovalEntity.getId().getAnnotationId())) { - log.info("hard delete ManualRedactions for file {} and annotation {}", idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId()); - manualRedactionProviderService.hardDeleteManualRedactions(idRemovalEntity.getId().getFileId(), idRemovalEntity.getId().getAnnotationId()); - } - } - - } - - - private boolean manualAddRedactionsContains(List addRedactions, String annotationId) { - - for (ManualRedactionEntryEntity manualRedactionEntryEntity : addRedactions) { - if (StringUtils.equals(annotationId, manualRedactionEntryEntity.getId().getAnnotationId())) { - return true; - } - } - return false; - } - -} - - - - - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DictionaryToEntityMigration2.java deleted file mode 100644 index 8c0f9f029..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DictionaryToEntityMigration2.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class DictionaryToEntityMigration2 extends Migration { - - private static final String NAME = "Migrate RedactionLogs remove false_positive and recommendation_ prefix"; - private static final long VERSION = 2; - - @Autowired - private DossierPersistenceService dossierPersistenceService; - - @Autowired - private FileStatusPersistenceService fileStatusPersistenceService; - - @Autowired - private FileManagementStorageService fileManagementStorageService; - - @Autowired - private ObjectMapper objectMapper; - - - public DictionaryToEntityMigration2() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var dossiers = dossierPersistenceService.findAllDossiers(); - dossiers.forEach(dossier -> { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); - log.info("Start migration of dossier {}", dossier.getId()); - files.forEach(file -> { - log.info("Start migration of file {}", file.getId()); - if (file.getHardDeletedTime() == null) { - - try { - var newRedactionLogEntries = new ArrayList(); - var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); - redactionLog.getRedactionLogEntry().forEach(entry -> { - - if (entry.getType().equals("false_positive")) { - log.info("skipping false_positive for dossier {} and file {}", dossier.getId(), file.getId()); - return; - } - - if (entry.getType().startsWith("recommendation_")) { - entry.setType(entry.getType().substring(15)); - entry.setRecommendation(true); - log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId()); - } - - newRedactionLogEntries.add(entry); - }); - - if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) { - redactionLog.setRedactionLogEntry(newRedactionLogEntries); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog); - log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); - } - } catch (Exception e) { - log.info("redactionLog {} does not exsist", file.getId()); - } - } - }); - }); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/EntityTypesMigration4.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/EntityTypesMigration4.java deleted file mode 100644 index e54b1664f..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/EntityTypesMigration4.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.List; -import java.util.Set; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class EntityTypesMigration4 extends Migration { - - private static final String NAME = "Set flags systemManaged, autoHideSkipped and hasDictionary for all entity types"; - private static final long VERSION = 4; - - @Autowired - private DictionaryPersistenceService dictionaryPersistenceService; - - @Autowired - private EntryPersistenceService entryPersistenceService; - - - public EntityTypesMigration4() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - Set systemManagedTypes = Set.of("ocr", "image", "logo", "signature", "formula", "imported_redaction", "dossier_redaction"); - Set autoHideSkippedTypes = Set.of("imported_redaction"); - - log.info("Will check all type entities and set systemManaged flag true for {}", systemManagedTypes); - log.info("Will check all type entities and set autoHideSkipped flag true for {}", autoHideSkippedTypes); - log.info("Will check all type entities and set hasDictionary flag true for types with entities"); - - List types = dictionaryPersistenceService.getAllTypes(false); - - for (TypeEntity type : types) { - if (type != null) { - - // Check if type is systemManaged - type.setSystemManaged(type.getType() != null && systemManagedTypes.contains(type.getType())); - - // Check if type is autoHideSkipped - type.setAutoHideSkipped(type.getType() != null && autoHideSkippedTypes.contains(type.getType())); - - // Check if type has dictionaries - List entries = entryPersistenceService.getEntries(type.getId(), DictionaryEntryType.ENTRY, null); - - type.setHasDictionary(entries != null && !entries.isEmpty()); - - } - } - - log.info("Save all types"); - dictionaryPersistenceService.saveAllTypes(types); - - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/FileSizeMigration8.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/FileSizeMigration8.java deleted file mode 100644 index 9992232e8..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/FileSizeMigration8.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class - -FileSizeMigration8 extends Migration { - - private static final String NAME = "Update file size"; - private static final long VERSION = 8; - - @Autowired - private FileStatusPersistenceService fileStatusPersistenceService; - - @Autowired - private FileManagementStorageService fileManagementStorageService; - - - public FileSizeMigration8() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - List allFiles = fileStatusPersistenceService.getAllFiles(); - - log.info("Number of all files: {}", allFiles.size()); - - allFiles.forEach(file -> { - // not hard deleted - if (file.getHardDeletedTime() == null) { - try { - var originFile = fileManagementStorageService.getStoredObjectBytes(file.getDossierId(), file.getId(), FileType.ORIGIN); - fileStatusPersistenceService.updateFileSize(file.getId(), originFile.length); - } catch (Exception e) { - log.warn("Failed to load file bytes for file: {} in dossier {}. Error message: {} ", file.getId(), file.getDossierId(), e.getMessage()); - } - } - }); - - } - -} - - - - - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/IndexMigration1.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/IndexMigration1.java deleted file mode 100644 index da5fb1e3e..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/IndexMigration1.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class IndexMigration1 extends Migration { - - private static final String NAME = "Recreate index and index all files"; - private static final long VERSION = 1; - @Autowired - private IndexingService indexingService; - - - public IndexMigration1() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files"); - indexingService.reindex(null, null, true); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeMigration9.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeMigration9.java deleted file mode 100644 index a5bbcad6f..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeMigration9.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class ManualRedactionTypeMigration9 extends Migration { - - private static final String NAME = "Add Manual Redaction Type"; - private static final long VERSION = 9; - private static final String MANUAL_TYPE = "manual"; - - @Autowired - private DictionaryPersistenceService dictionaryPersistenceService; - @Autowired - private DossierTemplateRepository dossierTemplateRepository; - @Autowired - private TypeRepository typeRepository; - - - public ManualRedactionTypeMigration9() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var allDossierTemplates = dossierTemplateRepository.findAll(); - - allDossierTemplates.forEach(dossierTemplateEntity -> { - - var entitiesForDossierTemplate = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateEntity.getId(), true); - - var manualType = entitiesForDossierTemplate.stream().filter(e -> MANUAL_TYPE.equals(e.getType())).findAny(); - manualType.ifPresent(typeEntity -> typeRepository.deleteById(typeEntity.getId())); - - var rank = 1; - for (var entity : entitiesForDossierTemplate) { - if (entity.getRank() > rank) { - rank = entity.getRank(); - } - } - rank += 1000; - - dictionaryPersistenceService.addType(MANUAL_TYPE, - dossierTemplateEntity.getId(), - "#9398a0", - "#c5d3eb", - "#c498fa", - rank, - false, - false, - false, - "Manual Redactions", - false, - "Manual Redactions", - null, - false, - true, - false, - true); - - }); - - } - -} - - - - - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateHighlights3.java deleted file mode 100644 index cb49dc667..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateHighlights3.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.io.ByteArrayInputStream; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class MigrateHighlights3 extends Migration { - - private static final String NAME = "Migrate Highlights"; - private static final long VERSION = 3; - - @Autowired - private DossierPersistenceService dossierPersistenceService; - - @Autowired - private FileStatusPersistenceService fileStatusPersistenceService; - - @Autowired - private FileManagementStorageService fileManagementStorageService; - - @Autowired - private FileStatusService fileStatusService; - - - public MigrateHighlights3() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var dossiers = dossierPersistenceService.findAllDossiers(); - dossiers.forEach(dossier -> { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); - files.forEach(file -> { - if (file.getHardDeletedTime() == null) { - - try { - if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { - var untouchedExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.UNTOUCHED); - - if (!untouchedExists) { - var originExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN); - - if (!originExists) { - log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId()); - return; - } - - fileManagementStorageService.storeObject(dossier.getId(), - file.getId(), - FileType.UNTOUCHED, - new ByteArrayInputStream(fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN))); - - } - - fileStatusService.addToPreprocessingQueue(file.getId(), file.getDossierId(), file.getFilename()); - - } else { - log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); - } - } catch (Exception e) { - log.warn("Failed to extract text highlights for document: {}", file.getId()); - } - } - }); - }); - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RemoveFalsePositiveManualRedactions6.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RemoveFalsePositiveManualRedactions6.java deleted file mode 100644 index 4cfe4e15e..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RemoveFalsePositiveManualRedactions6.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class RemoveFalsePositiveManualRedactions6 extends Migration { - - private static final String NAME = "Remove false positives manual redactions"; - private static final long VERSION = 6; - - @Autowired - private DossierPersistenceService dossierPersistenceService; - - @Autowired - private FileStatusPersistenceService fileStatusPersistenceService; - - @Autowired - private ManualRedactionService manualRedactionService; - - @Autowired - private AddRedactionPersistenceService addRedactionPersistenceService; - - - public RemoveFalsePositiveManualRedactions6() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var dossiers = dossierPersistenceService.findAllDossiers(); - dossiers.forEach(dossier -> { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); - files.forEach(file -> { - if (file.getHardDeletedTime() == null) { - var annotationIdsToRemove = new ArrayList(); - var manualRedactions = manualRedactionService.getManualRedactions( file.getId()); - - if (manualRedactions != null && manualRedactions.getEntriesToAdd() != null) { - manualRedactions.getEntriesToAdd().forEach(addRedaction -> { - if (addRedaction.getType().contains("false_positive")) { - annotationIdsToRemove.add(addRedaction.getAnnotationId()); - } - }); - } - - log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(), annotationIdsToRemove); - - if (!annotationIdsToRemove.isEmpty()) { - annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id)); - } - } - }); - }); - } - -} - - - - - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/SimplifiedTextMigration12.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/SimplifiedTextMigration12.java deleted file mode 100644 index 1040914bc..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/SimplifiedTextMigration12.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.model.simplifiedtext.Text; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; -import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.Setter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class SimplifiedTextMigration12 extends Migration { - - private static final String NAME = "Add simplified text file if missing"; - private static final long VERSION = 12; - - @Autowired - private DossierPersistenceService dossierPersistenceService; - - @Autowired - private FileStatusPersistenceService fileStatusPersistenceService; - - @Autowired - private StorageService storageService; - - - public SimplifiedTextMigration12() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var dossiers = dossierPersistenceService.findAllDossiers(); - dossiers.forEach(dossier -> { - if (dossier.getHardDeletedTime() == null) { - var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); - log.info("Start migration of dossier {}", dossier.getId()); - files.forEach(file -> { - if (file.getHardDeletedTime() == null) { - log.info("Start migration of file {}", file.getId()); - migrateFile(dossier.getId(), file.getId()); - log.info("Finished migration of file {}", file.getId()); - } - }); - log.info("Finished migration of dossier {}", dossier.getId()); - } - }); - } - - - @SneakyThrows - public void migrateFile(String dossierId, String fileId) { - - try { - - if (!storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT))) { - var text = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class); - storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT), text); - } - } catch (StorageObjectDoesNotExist e) { - log.warn("Text not found for dossier {} and file {}, ignoring....", dossierId, fileId); - } - } - -} - - - - - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/TypeToEntityMigration5.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/TypeToEntityMigration5.java deleted file mode 100644 index d7348c5c2..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/TypeToEntityMigration5.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; -import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; -import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Service -public class TypeToEntityMigration5 extends Migration { - - private static final String NAME = "Migrate false positives, set recommendation color, remove unneeded dictionaries"; - private static final long VERSION = 5; - - @Autowired - private DictionaryPersistenceService dictionaryPersistenceService; - - @Autowired - private DossierTemplateRepository dossierTemplateRepository; - - @Autowired - private EntryPersistenceService entryPersistenceService; - - @Autowired - private DictionaryManagementService dictionaryManagementService; - - - public TypeToEntityMigration5() { - - super(NAME, VERSION); - } - - - @Override - protected void migrate() { - - var types = dictionaryPersistenceService.getAllTypes(false); - types.forEach(type -> type.setRecommendationHexColor("#8df06c")); - dictionaryPersistenceService.saveAllTypes(types); - - var dossierTemplates = dossierTemplateRepository.findAllWhereDeletedIsFalse(); - - dossierTemplates.forEach(dossierTemplate -> { - log.info("Starting false positive migration of dossierTemplate {}", dossierTemplate.getId()); - var typeIdsToDelete = new HashSet(); - var typesOfDossierTemplate = dictionaryPersistenceService.getAllTypes(false); - var falsePositive = typesOfDossierTemplate.stream().filter(t -> t.getType().equals("false_positive")).findFirst(); - if (falsePositive.isEmpty()) { - log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName()); - return; - } - typeIdsToDelete.add(falsePositive.get().getId()); - var falsePositiveEntries = entryPersistenceService.getEntries(falsePositive.get().getId(), DictionaryEntryType.ENTRY, null) - .stream() - .map(BaseDictionaryEntry::getValue) - .collect(Collectors.toSet()); - log.info("False positive migration of dossierTemplate {} has {} false positive entries", dossierTemplate.getId(), falsePositiveEntries.size()); - - typesOfDossierTemplate.stream().filter(t -> !t.getType().equals("false_positive")).forEach(typeEntity -> { - log.info("Start processing type {}", typeEntity.getType()); - - if (typeEntity.getType().startsWith("recommendation_")) { - typeIdsToDelete.add(typeEntity.getId()); - log.info("Finished processing type {}", typeEntity.getType()); - return; - } - - var entries = entryPersistenceService.getEntries(typeEntity.getId(), DictionaryEntryType.ENTRY, null) - .stream() - .map(BaseDictionaryEntry::getValue) - .collect(Collectors.toSet()); - log.info("type {} has {} entries", typeEntity.getType(), entries); - - var typeFalsePositives = new HashSet(); - falsePositiveEntries.forEach(falsePositiveValue -> { - if (containsAnyWord(falsePositiveValue, entries, typeEntity.isCaseInsensitive())) { - typeFalsePositives.add(falsePositiveValue); - } - }); - - if (!typeFalsePositives.isEmpty()) { - dictionaryManagementService.addEntries(typeEntity.getId(), new ArrayList<>(typeFalsePositives), false, false, DictionaryEntryType.FALSE_POSITIVE); - log.info("Added {} for type {}", typeFalsePositives.size(), typeEntity.getType()); - } - log.info("Finished processing type {}", typeEntity.getType()); - }); - - typeIdsToDelete.forEach(typeIdToDelete -> dictionaryManagementService.deleteType(typeIdToDelete)); - log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId()); - }); - - } - - - public boolean containsAnyWord(String text, Set values, boolean isCaseInsensitive) { - - String inputString = isCaseInsensitive ? text.toLowerCase(Locale.ROOT) : text; - - for (String value : values) { - - String cleanValue = isCaseInsensitive ? value.toLowerCase(Locale.ROOT).trim() : value.trim(); - - int startIndex; - int stopIndex = 0; - do { - startIndex = inputString.indexOf(cleanValue, stopIndex); - stopIndex = startIndex + cleanValue.length(); - - if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator( - inputString.charAt(stopIndex)))) { - return true; - } - } while (startIndex > -1); - } - return false; - } - - - private boolean isSeparator(char c) { - - return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’'; - } - -}