diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java index 51de764f3..c7afaa23f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java @@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.audit.Au import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository; import org.springframework.boot.autoconfigure.domain.EntityScan; @@ -19,7 +20,7 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories; @Configuration @ComponentScan -@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class}) +@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class, MigrationEntity.class}) @EnableJpaRepositories(basePackageClasses = ColorsRepository.class) @EnableFeignClients(basePackageClasses = {PDFTronRedactionClient.class}) public class PersistenceServiceProcessorConfiguration { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java new file mode 100644 index 000000000..f82bf9844 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.migration; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@Entity +@Table(name = "migration") +public class MigrationEntity { + + @Id + @GeneratedValue + private long id; + + @Column + private String name; + + @Column + private long version; +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java new file mode 100644 index 000000000..87e2b1956 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java @@ -0,0 +1,51 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence; + +import java.util.Comparator; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.MigrationRepository; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class MigrationPersistenceService { + + private final MigrationRepository migrationRepository; + + + @Transactional + public void insertMigration(String name, long version) { + + migrationRepository.save(MigrationEntity.builder().name(name).version(version).build()); + } + + + public boolean isProcessed(long version, long lastProcessedVersion) { + + var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion); + if (migrations == null || migrations.isEmpty()) { + if(version > lastProcessedVersion){ + return false; + } + return true; + } + + return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent(); + } + + + public Long getLatestProcessedVersion() { + + var migrations = migrationRepository.findAll(); + if (migrations == null || migrations.isEmpty()) { + return null; + } + return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/MigrationRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/MigrationRepository.java new file mode 100644 index 000000000..58ddbfebe --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/MigrationRepository.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity; + +public interface MigrationRepository extends JpaRepository { + + List findByVersionGreaterThan(long startVersion); + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java new file mode 100644 index 000000000..90ec3e2df --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.peristence.v1.server.migration; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@RequiredArgsConstructor +public abstract class Migration { + + private final String name; + private final long version; + + @Autowired + @Setter + private MigrationPersistenceService migrationPersistenceService; + + + public void run() { + + var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion(); + if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) { + log.info("Starting migration with name {} and version {}", name, version); + migrate(); + migrationPersistenceService.insertMigration(name, version); + log.info("Finished migration with name {} and version {}", name, version); + } else { + log.info("Migration with name: {} and version {} already done, skipping....", name, version); + } + } + + + protected abstract void migrate(); + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java new file mode 100644 index 000000000..66fd349e4 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.peristence.v1.server.migration; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.peristence.v1.server.migration.migrations.DictionaryToEntityMigration2; +import com.iqser.red.service.peristence.v1.server.migration.migrations.IndexMigration1; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class MigrationStarterService { + + // This should always be the latest migration version (2), but as 3.2.x does not have this, currently it is 1. + // After migration to 3.3.x we must fix this. + public static final long MIGRATION_SEED_VERSION = 1; + + private final IndexMigration1 indexMigration1; + private final DictionaryToEntityMigration2 dictionaryToEntityMigration2; + private final FileManagementServiceSettings settings; + private final ApplicationContext ctx; + private final MigrationPersistenceService migrationPersistenceService; + + + @EventListener(ApplicationReadyEvent.class) + public void migrate() { + + //This is always running on startup + seedMigration(); + + // This should only run in post upgrade hook + if (settings.isMigrateOnly()) { + + indexMigration1.run(); + dictionaryToEntityMigration2.run(); + + System.exit(SpringApplication.exit(ctx, () -> 0)); + } + } + + + private void seedMigration(){ + if(migrationPersistenceService.getLatestProcessedVersion() == null){ + migrationPersistenceService.insertMigration("migration start version" , MIGRATION_SEED_VERSION); + } + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/index/MigrateIndexServiceStarter.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/index/MigrateIndexServiceStarter.java deleted file mode 100644 index 2e6f61b23..000000000 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/index/MigrateIndexServiceStarter.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.iqser.red.service.peristence.v1.server.migration.index; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationContext; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.peristence.v1.server.service.IndexingService; -import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class MigrateIndexServiceStarter { - - private final IndexingService indexingService; - private final FileManagementServiceSettings settings; - private final ApplicationContext ctx; - - - @EventListener(ApplicationReadyEvent.class) - public void reindexFiles() { - - if (settings.isDropIndexAndReindexFiles()) { - log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files"); - indexingService.reindex(null, null, true); - - System.exit(SpringApplication.exit(ctx, () -> 0)); - } - } - -} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java new file mode 100644 index 000000000..05f701e2f --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -0,0 +1,67 @@ +package com.iqser.red.service.peristence.v1.server.migration.migrations; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.peristence.v1.server.migration.Migration; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; + +import lombok.Setter; + +@Setter +@Service +public class DictionaryToEntityMigration2 extends Migration { + + private static final String NAME = "Migrate RedactionLogs remove false_positive and recommendation_ prefix"; + private static final long VERSION = 2; + + @Autowired + private DossierPersistenceService dossierPersistenceService; + + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + + @Autowired + private FileManagementStorageService fileManagementStorageService; + + @Autowired + private ObjectMapper objectMapper; + + public DictionaryToEntityMigration2() { + + super(NAME, VERSION); + } + + + @Override + protected void migrate() { + var dossiers = dossierPersistenceService.findAllDossiers(); + dossiers.forEach(dossier -> { + var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()); + files.forEach(file -> { + if(file.getDeleted() != null){ + var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); + redactionLog.getRedactionLogEntry().removeIf(entry -> entry.getType().equals("false_positive")); + redactionLog.getRedactionLogEntry().forEach(entry -> { + if(entry.getType().startsWith("recommendation_")){ + entry.setType(entry.getType().substring(15)); + entry.setRecommendation(true); + } + }); + try { + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); + } catch (JsonProcessingException e) { + throw new RuntimeException("Migration failed"); + } + + } + }); + }); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java new file mode 100644 index 000000000..b547da4fb --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.peristence.v1.server.migration.migrations; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.peristence.v1.server.migration.Migration; +import com.iqser.red.service.peristence.v1.server.service.IndexingService; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Setter +@Service +public class IndexMigration1 extends Migration { + + private static final String NAME = "Recreate index and index all files"; + private static final long VERSION = 1; + + + public IndexMigration1() { + + super(NAME, VERSION); + } + + + @Autowired + private IndexingService indexingService; + + + @Override + protected void migrate() { + + log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files"); + indexingService.reindex(null, null, true); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java index 1573d66cf..1e381fd52 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java @@ -21,10 +21,10 @@ public class FileManagementServiceSettings { private int softDeleteCleanupTime = 96; - private boolean dropIndexAndReindexFiles; - private boolean imageServiceEnabled = true; private boolean nerServiceEnabled = true; private boolean storeImageFile = true; + + private boolean migrateOnly; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/18-add-migration-table.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/18-add-migration-table.yaml new file mode 100644 index 000000000..a1b3c1d3a --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/18-add-migration-table.yaml @@ -0,0 +1,21 @@ +databaseChangeLog: + - changeSet: + id: add-migration-table + author: dom + changes: + - createTable: + columns: + - column: + constraints: + nullable: false + primaryKey: true + primaryKeyName: migration_pkey + name: id + type: BIGINT + - column: + name: name + type: VARCHAR(255) + - column: + name: version + type: BIGINT + tableName: migration \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 7ba172e7b..08a3de2b9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -40,4 +40,6 @@ databaseChangeLog: - include: file: db/changelog/16-added-has-dictionary-to-entities.changelog.yaml - include: - file: db/changelog/17-digital-signature-kms.changelog.yaml \ No newline at end of file + file: db/changelog/17-digital-signature-kms.changelog.yaml + - include: + file: db/changelog/18-add-migration-table.yaml \ No newline at end of file