diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java index d6912d9f5..f78874144 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java @@ -5,15 +5,20 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.nio.file.Files; import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.concurrent.atomic.AtomicInteger; import org.springframework.stereotype.Service; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageClientCache; import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.service.azure.AzureBlobClient; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -107,9 +112,90 @@ public class UncompressedFilesMigrationService { log.info("Key: {} migrated successfully", key); } - } else { - throw new StorageException("No client available for tenant: " + tenant); + } else if (client.getAzureBlobClient() != null) { + + var azureClient = client.getAzureBlobClient(); + var blobContainerClient = + azureClient.getBlobServiceClient().getBlobContainerClient(client.getAzureBlobClient().getAzureStorageConnection().getContainerName()); + + List keysToMigrate = new ArrayList<>(); + + var counter = new AtomicInteger(); + for (var item : blobContainerClient.listBlobs()) { + + counter.incrementAndGet(); + if (!item.getName().toLowerCase(Locale.ROOT).endsWith(".gz")) { + log.info("Found key to migrate/compress: {}", item.getName()); + keysToMigrate.add(item.getName()); + } + } + + log.info("Total files that require compression: {} / {} ", keysToMigrate.size(), counter.get()); + + + int attempts = 0; + do { + keysToMigrate = migrateBlobKeys(tenant, keysToMigrate, azureClient); + attempts++; + } while (!keysToMigrate.isEmpty() && attempts <= 3); + + if (!keysToMigrate.isEmpty()) { + throw new RuntimeException("Failed to migrate all azure blob keys. Remaining: " + keysToMigrate.size()); + } + } } + + @SneakyThrows + private List migrateBlobKeys(String tenant, List keysToMigrate, AzureBlobClient azureBlobClient) { + + String tmpdir = Files.createTempDirectory("compressed-migration").toFile().getAbsolutePath(); + + var failedKeys = new ArrayList(); + for (var key : keysToMigrate) { + + try { + var blobContainerClient = + azureBlobClient.getBlobServiceClient().getBlobContainerClient(azureBlobClient.getAzureStorageConnection().getContainerName()); + + log.info("Migrating key: {}", key); + + var itemClient = blobContainerClient.getBlobClient(key); + + var tempFile = new File(tmpdir, key); + // in case it was created by a previous migration + tempFile.mkdirs(); + tempFile.delete(); + + itemClient.downloadToFile(tempFile.getAbsolutePath(), true); + + var fis = new FileInputStream(tempFile); + storageService.storeObject(tenant, key, fis); + + IOUtils.closeQuietly(fis); + try { + tempFile.delete(); + } catch (Exception e) { + log.debug("Failed to delete temp file: {}", tempFile.getAbsolutePath()); + } + + try { + BlobClient blobClient = blobContainerClient.getBlobClient(key); + blobClient.delete(); + } catch (Exception e) { + log.debug("Failed to delete item {}", key); + } + + log.info("Key: {} migrated successfully", key); + + } catch (Exception e) { + failedKeys.add(key); + } + } + + return failedKeys; + + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 6829aa72a..bd46690d2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -127,6 +127,8 @@ databaseChangeLog: file: db/changelog/tenant/sql/47-add-keep_hidden_text.sql - include: file: db/changelog/tenant/sql/203-acl-duplicate-cleanup.sql + - include: + file: db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql - include: file: db/changelog/tenant/sql/203-spring-acl-constraints.changelog.sql - include: @@ -170,4 +172,4 @@ databaseChangeLog: - include: file: db/changelog/tenant/115-add-saas-migration-status-table.yaml - include: - file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml \ No newline at end of file + file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql new file mode 100644 index 000000000..a52e30cfb --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/203-migration-fix-acl-duplicate-cleanup.sql @@ -0,0 +1 @@ +delete from acl_entry s1 where s1.id != ( select min(s2.id) from acl_entry s2 where s2.acl_object_identity = s1.acl_object_identity and s2.ace_order = s1.ace_order);