Pull request #322: RED-2836: Migrate redactionlogs for dictionary to entity change

Merge in RED/persistence-service from RED-2836 to master

* commit '69afa534f0edeb022e17d44ada222ecd055306f2':
  RED-2836: Migrate redactionlogs for dictionary to entity change

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java

@@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.audit.Au
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
 import org.springframework.boot.autoconfigure.domain.EntityScan;
@@ -19,7 +20,7 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 
 @Configuration
 @ComponentScan
-@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class})
+@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class, MigrationEntity.class})
 @EnableJpaRepositories(basePackageClasses = ColorsRepository.class)
 @EnableFeignClients(basePackageClasses = {PDFTronRedactionClient.class})
 public class PersistenceServiceProcessorConfiguration {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java

@@ -0,0 +1,31 @@
+package com.iqser.red.service.persistence.management.v1.processor.entity.migration;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "migration")
+public class MigrationEntity {
+
+    @Id
+    @GeneratedValue
+    private long id;
+
+    @Column
+    private String name;
+
+    @Column
+    private long version;
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java

@@ -0,0 +1,51 @@
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.MigrationRepository;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class MigrationPersistenceService {
+
+    private final MigrationRepository migrationRepository;
+
+
+    @Transactional
+    public void insertMigration(String name, long version) {
+
+        migrationRepository.save(MigrationEntity.builder().name(name).version(version).build());
+    }
+
+
+    public boolean isProcessed(long version, long lastProcessedVersion) {
+
+        var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion);
+        if (migrations == null || migrations.isEmpty()) {
+            if(version > lastProcessedVersion){
+                return false;
+            }
+            return true;
+        }
+
+        return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent();
+    }
+
+
+    public Long getLatestProcessedVersion() {
+
+        var migrations = migrationRepository.findAll();
+        if (migrations == null || migrations.isEmpty()) {
+            return null;
+        }
+        return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/MigrationRepository.java

@@ -0,0 +1,13 @@
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
+
+import java.util.List;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity;
+
+public interface MigrationRepository extends JpaRepository<MigrationEntity, String> {
+
+    List<MigrationEntity> findByVersionGreaterThan(long startVersion);
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java

@@ -0,0 +1,41 @@
+package com.iqser.red.service.peristence.v1.server.migration;
+
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService;
+
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Data
+@Slf4j
+@RequiredArgsConstructor
+public abstract class Migration {
+
+    private final String name;
+    private final long version;
+
+    @Autowired
+    @Setter
+    private MigrationPersistenceService migrationPersistenceService;
+
+
+    public void run() {
+
+        var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion();
+        if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) {
+            log.info("Starting migration with name {} and version {}", name, version);
+            migrate();
+            migrationPersistenceService.insertMigration(name, version);
+            log.info("Finished migration with name {} and version {}", name, version);
+        } else {
+            log.info("Migration with name: {} and version {} already done, skipping....", name, version);
+        }
+    }
+
+
+    protected abstract void migrate();
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.peristence.v1.server.migration;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.peristence.v1.server.migration.migrations.DictionaryToEntityMigration2;
+import com.iqser.red.service.peristence.v1.server.migration.migrations.IndexMigration1;
+import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class MigrationStarterService {
+
+    // This should always be the latest migration version (2), but as 3.2.x does not have this, currently it is 1.
+    // After migration to 3.3.x we must fix this.
+    public static final long MIGRATION_SEED_VERSION = 1;
+
+    private final IndexMigration1 indexMigration1;
+    private final DictionaryToEntityMigration2 dictionaryToEntityMigration2;
+    private final FileManagementServiceSettings settings;
+    private final ApplicationContext ctx;
+    private final MigrationPersistenceService migrationPersistenceService;
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void migrate() {
+
+        //This is always running on startup
+        seedMigration();
+
+        // This should only run in post upgrade hook
+        if (settings.isMigrateOnly()) {
+
+            indexMigration1.run();
+            dictionaryToEntityMigration2.run();
+
+            System.exit(SpringApplication.exit(ctx, () -> 0));
+        }
+    }
+
+
+    private void seedMigration(){
+        if(migrationPersistenceService.getLatestProcessedVersion() == null){
+            migrationPersistenceService.insertMigration("migration start version" , MIGRATION_SEED_VERSION);
+        }
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/index/MigrateIndexServiceStarter.java

@@ -1,36 +0,0 @@
-package com.iqser.red.service.peristence.v1.server.migration.index;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.peristence.v1.server.service.IndexingService;
-import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class MigrateIndexServiceStarter {
-
-    private final IndexingService indexingService;
-    private final FileManagementServiceSettings settings;
-    private final ApplicationContext ctx;
-
-
-    @EventListener(ApplicationReadyEvent.class)
-    public void reindexFiles() {
-
-        if (settings.isDropIndexAndReindexFiles()) {
-            log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files");
-            indexingService.reindex(null, null, true);
-
-            System.exit(SpringApplication.exit(ctx, () -> 0));
-        }
-    }
-
-}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java

@@ -0,0 +1,67 @@
+package com.iqser.red.service.peristence.v1.server.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.peristence.v1.server.migration.Migration;
+import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+
+import lombok.Setter;
+
+@Setter
+@Service
+public class DictionaryToEntityMigration2 extends Migration {
+
+    private static final String NAME = "Migrate RedactionLogs remove false_positive and recommendation_ prefix";
+    private static final long VERSION = 2;
+
+    @Autowired
+    private DossierPersistenceService dossierPersistenceService;
+
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
+    @Autowired
+    private FileManagementStorageService fileManagementStorageService;
+
+    @Autowired
+    private ObjectMapper objectMapper;
+
+    public DictionaryToEntityMigration2() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+        var dossiers = dossierPersistenceService.findAllDossiers();
+        dossiers.forEach(dossier -> {
+            var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
+            files.forEach(file -> {
+                if(file.getDeleted() != null){
+                    var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
+                    redactionLog.getRedactionLogEntry().removeIf(entry -> entry.getType().equals("false_positive"));
+                    redactionLog.getRedactionLogEntry().forEach(entry -> {
+                        if(entry.getType().startsWith("recommendation_")){
+                            entry.setType(entry.getType().substring(15));
+                            entry.setRecommendation(true);
+                        }
+                    });
+                    try {
+                        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
+                    } catch (JsonProcessingException e) {
+                        throw new RuntimeException("Migration failed");
+                    }
+
+                }
+            });
+        });
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java

@@ -0,0 +1,38 @@
+package com.iqser.red.service.peristence.v1.server.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.peristence.v1.server.migration.Migration;
+import com.iqser.red.service.peristence.v1.server.service.IndexingService;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class IndexMigration1 extends Migration {
+
+    private static final String NAME = "Recreate index and index all files";
+    private static final long VERSION = 1;
+
+
+    public IndexMigration1() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Autowired
+    private IndexingService indexingService;
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Will call SearchService via queue to close, drop, recreate index and reindex all files");
+        indexingService.reindex(null, null, true);
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java

@@ -21,10 +21,10 @@ public class FileManagementServiceSettings {
 
     private int softDeleteCleanupTime = 96;
 
-    private boolean dropIndexAndReindexFiles;
-
     private boolean imageServiceEnabled = true;
     private boolean nerServiceEnabled = true;
 
     private boolean storeImageFile = true;
+
+    private boolean migrateOnly;
 }

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/18-add-migration-table.yaml

@@ -0,0 +1,21 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-migration-table
+      author: dom
+      changes:
+        - createTable:
+            columns:
+              - column:
+                  constraints:
+                    nullable: false
+                    primaryKey: true
+                    primaryKeyName: migration_pkey
+                  name: id
+                  type: BIGINT
+              - column:
+                  name: name
+                  type: VARCHAR(255)
+              - column:
+                  name: version
+                  type: BIGINT
+            tableName: migration

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -41,3 +41,5 @@ databaseChangeLog:
       file: db/changelog/16-added-has-dictionary-to-entities.changelog.yaml
   - include:
       file: db/changelog/17-digital-signature-kms.changelog.yaml
+  - include:
+      file: db/changelog/18-add-migration-table.yaml