Merge branch 'RED-6661-fixes' into 'master'

migration for uncompressed azure blobs

Closes RED-6661

See merge request redactmanager/persistence-service!264

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java

@@ -5,15 +5,20 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.springframework.stereotype.Service;
 
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
 import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageClientCache;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.service.azure.AzureBlobClient;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -107,9 +112,90 @@ public class UncompressedFilesMigrationService {
                 log.info("Key: {} migrated successfully", key);
             }
 
-        } else {
-            throw new StorageException("No client available for tenant: " + tenant);
+        } else if (client.getAzureBlobClient() != null) {
+
+            var azureClient = client.getAzureBlobClient();
+            var blobContainerClient =
+                    azureClient.getBlobServiceClient().getBlobContainerClient(client.getAzureBlobClient().getAzureStorageConnection().getContainerName());
+
+            List<String> keysToMigrate = new ArrayList<>();
+
+            var counter = new AtomicInteger();
+            for (var item : blobContainerClient.listBlobs()) {
+
+                counter.incrementAndGet();
+                if (!item.getName().toLowerCase(Locale.ROOT).endsWith(".gz")) {
+                    log.info("Found key to migrate/compress: {}", item.getName());
+                    keysToMigrate.add(item.getName());
+                }
+            }
+
+            log.info("Total files that require compression: {}  / {} ", keysToMigrate.size(), counter.get());
+
+
+            int attempts = 0;
+            do {
+                keysToMigrate = migrateBlobKeys(tenant, keysToMigrate, azureClient);
+                attempts++;
+            } while (!keysToMigrate.isEmpty() && attempts <= 3);
+
+            if (!keysToMigrate.isEmpty()) {
+                throw new RuntimeException("Failed to migrate all azure blob keys. Remaining: " + keysToMigrate.size());
+            }
+
         }
     }
 
+
+    @SneakyThrows
+    private List<String> migrateBlobKeys(String tenant, List<String> keysToMigrate, AzureBlobClient azureBlobClient) {
+
+        String tmpdir = Files.createTempDirectory("compressed-migration").toFile().getAbsolutePath();
+
+        var failedKeys = new ArrayList<String>();
+        for (var key : keysToMigrate) {
+
+            try {
+                var blobContainerClient =
+                        azureBlobClient.getBlobServiceClient().getBlobContainerClient(azureBlobClient.getAzureStorageConnection().getContainerName());
+
+                log.info("Migrating key: {}", key);
+
+                var itemClient = blobContainerClient.getBlobClient(key);
+
+                var tempFile = new File(tmpdir, key);
+                // in case it was created by a previous migration
+                tempFile.mkdirs();
+                tempFile.delete();
+
+                itemClient.downloadToFile(tempFile.getAbsolutePath(), true);
+
+                var fis = new FileInputStream(tempFile);
+                storageService.storeObject(tenant, key, fis);
+
+                IOUtils.closeQuietly(fis);
+                try {
+                    tempFile.delete();
+                } catch (Exception e) {
+                    log.debug("Failed to delete temp file: {}", tempFile.getAbsolutePath());
+                }
+
+                try {
+                    BlobClient blobClient = blobContainerClient.getBlobClient(key);
+                    blobClient.delete();
+                } catch (Exception e) {
+                    log.debug("Failed to delete item {}", key);
+                }
+
+                log.info("Key: {} migrated successfully", key);
+
+            } catch (Exception e) {
+                failedKeys.add(key);
+            }
+        }
+
+        return failedKeys;
+
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml

@@ -127,6 +127,8 @@ databaseChangeLog:
       file: db/changelog/tenant/sql/47-add-keep_hidden_text.sql
   - include:
       file: db/changelog/tenant/sql/203-acl-duplicate-cleanup.sql
+  - include:
+      file: db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql
   - include:
       file: db/changelog/tenant/sql/203-spring-acl-constraints.changelog.sql
   - include:
@@ -170,4 +172,4 @@ databaseChangeLog:
   - include:
       file: db/changelog/tenant/115-add-saas-migration-status-table.yaml
   - include:
-      file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml
+      file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql

@@ -0,0 +1 @@
+delete from acl_entry s1 where s1.id != ( select min(s2.id) from acl_entry s2 where s2.acl_object_identity = s1.acl_object_identity and s2.ace_order = s1.ace_order);