RED-4686 - storage commons update

persistence-service-v1/persistence-service-server-v1/pom.xml

@@ -186,6 +186,19 @@
     </dependencies>
     <build>
         <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <annotationProcessors>
+                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
+                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
+                    </annotationProcessors>
+                </configuration>
+            </plugin>
+
+
             <plugin>
                 <!-- generate git.properties for exposure in /info -->
                 <groupId>pl.project13.maven</groupId>

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java

@@ -120,10 +120,8 @@ public class ReanalysisController implements ReanalysisResource {
     public void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest) {
 
         var importedRedactions = fileManagementStorageService.getImportedRedactions(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId());
-        importedRedactions.getImportedRedactions().entrySet().forEach(entry -> {
+        importedRedactions.getImportedRedactions().forEach((key, value) -> value.removeIf(v -> deleteImportedRedactionsRequest.getAnnotationIds().contains(v.getId())));
-            entry.getValue().removeIf(v -> deleteImportedRedactionsRequest.getAnnotationIds().contains(v.getId()));
+        fileManagementStorageService.storeJSONObject(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), FileType.IMPORTED_REDACTIONS, importedRedactions);
-        });
-        fileManagementStorageService.storeObject(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), FileType.IMPORTED_REDACTIONS, objectMapper.writeValueAsBytes(importedRedactions));
         fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false);
     }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.controller;
 
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
@@ -55,7 +56,7 @@ public class ReportTemplateController implements ReportTemplateResource {
 
         String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest
                 .getFileName());
-        storageService.storeObject(storageId, reportTemplateUploadRequest.getTemplate());
+        storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate()));
 
         if (templateId != null) {
             reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java

@@ -75,12 +75,10 @@ public class DictionaryToEntityMigration2 extends Migration {
 
                         if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
                             redactionLog.setRedactionLogEntry(newRedactionLogEntries);
-                            fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
+                            fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
                             log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
                         }
-                    } catch (JsonProcessingException e) {
+                    } catch (Exception e) {
-                        throw new RuntimeException("Migration failed");
-                    } catch (NotFoundException e) {
                         log.info("redactionLog {} does not exsist", file.getId());
                     }
                 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations;
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
 import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
 import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
@@ -12,6 +11,8 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.io.ByteArrayInputStream;
+
 @Slf4j
 @Setter
 @Service
@@ -60,7 +61,7 @@ public class MigrateHighlights3 extends Migration {
                                 }
 
                                 fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
-                                        fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
+                                        new ByteArrayInputStream(fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)));
 
                             }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java

@@ -9,14 +9,13 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
+import liquibase.pro.packaged.T;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;
 
-import java.io.IOException;
 import java.io.InputStream;
 
 @Slf4j
@@ -24,7 +23,6 @@ import java.io.InputStream;
 @RequiredArgsConstructor
 public class FileManagementStorageService {
 
-    private final ObjectMapper objectMapper;
     private final StorageService storageService;
 
 
@@ -42,10 +40,14 @@ public class FileManagementStorageService {
         return IOUtils.toByteArray(storageService.getObject(storageId).getInputStream());
     }
 
+    public void storeObject(String dossierId, String fileId, FileType fileType, InputStream stream){
 
-    public void storeObject(String dossierId, String fileId, FileType fileType, byte[] data) {
+        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), stream);
+    }
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), data);
+    public <T> void storeJSONObject(String dossierId, String fileId, FileType fileType, T jsonObject){
+
+        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), jsonObject);
     }
 
 
@@ -56,45 +58,34 @@ public class FileManagementStorageService {
 
 
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
-
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             throw new NotFoundException("RedactionLog does not exist");
         }
-
-        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class);
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert Text", e);
-        }
     }
 
 
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
-        var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
         try {
-            return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
-        } catch (IOException e) {
+        } catch (StorageObjectDoesNotExist e) {
-            throw new RuntimeException("Could not convert RedactionLog", e);
+            log.debug("SectionGrid not available.");
+            throw new NotFoundException("RedactionLog does not exist");
+        } catch (Exception e) {
+            throw new RuntimeException("Could not convert SectionGrid", e);
         }
     }
 
 
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("ImportedRedactions does not exist");
-        }
+        } catch (Exception e) {
-
-        try {
-            return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
-        } catch (IOException e) {
             throw new RuntimeException("Could not convert ImportedRedactions", e);
         }
     }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ImageMessageReceiver.java

@@ -1,21 +1,21 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.springframework.amqp.core.Message;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.HashMap;
 
 @Slf4j
 @Service
@@ -33,14 +33,13 @@ public class ImageMessageReceiver {
     @RabbitListener(queues = MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE)
     public void receive(byte[] in) {
 
-        HashMap<String, Object> imageResponse = objectMapper.readValue(in, new TypeReference<>() {
+        JsonNode imageResponse = objectMapper.readTree(in);
-        });
 
-        String dossierId = (String) imageResponse.get("dossierId");
+        String dossierId = imageResponse.path("dossierId").asText();
-        String fileId = (String) imageResponse.get("fileId");
+        String fileId = imageResponse.path("fileId").asText();
 
-        if(settings.isStoreImageFile()) {
+        if (settings.isStoreImageFile()) {
-            fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, in);
+            fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, new ByteArrayInputStream(in));
         }
         fileStatusService.setStatusAnalyse(dossierId, fileId, false);
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java

@@ -18,6 +18,7 @@ import org.apache.commons.codec.binary.Base64;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.io.ByteArrayInputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
@@ -56,11 +57,11 @@ public class FileTesterAndProvider {
 
         assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
 
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, 1,
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, new RedactionLog(1, 1,
                 List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
-                null, 0, 0, 0, 0)));
+                null, 0, 0, 0, 0));
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid()));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, new SectionGrid());
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, objectMapper.writeValueAsBytes("bytes of the file"));
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, new ByteArrayInputStream(objectMapper.writeValueAsBytes("bytes of the file")));
 
         return file;
     }
@@ -71,7 +72,7 @@ public class FileTesterAndProvider {
         var fileId = Base64.encodeBase64String((dossier.getId() + fileName).getBytes(StandardCharsets.UTF_8));
 
         AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1");
-        fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
         JSONPrimitive<String> uploadResult = uploadClient.upload(upload, false);
 
         return uploadResult.getValue();

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -169,9 +169,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII");
 
         // assume file is already proccessed once, test that add to dict triggers reanalysis
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.TEXT, "{}");
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES,  "{}");
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO,  "{}");
         fileStatusPersistenceService.updateProcessingStatus(file.getId(), ProcessingStatus.PROCESSED);
 
         var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
@@ -308,7 +308,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .type("manual")
                 .value("value entry")
                 .build()), null, 0, 0, 0, 0);
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
 
         when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(redactionLog);
         manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(), UpdateRedactionRequest.builder()
@@ -334,7 +334,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedRemoveRedaction.getStatus()).isEqualTo(AnnotationStatus.DECLINED);
 
         var redLog = new RedactionLog(1, 1, List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()), null, 0, 0, 0, 0);
-        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redLog));
+        fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redLog);
 
         when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(redLog);
         var removeRedaction2 = manualRedactionClient.addRemoveRedaction(dossier.getId(), file.getId(), List.of(RemoveRedactionRequest.builder()

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.peristence.v1.server.integration.utils;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.S3StorageService;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -11,15 +14,17 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService implements StorageService {
 
-    private Map<String, File> dataMap = new HashMap<>();
+    private final Map<String, File> dataMap = new HashMap<>();
 
     public FileSystemBackedStorageService() {
-        super(null, null);
     }
 
     @SneakyThrows
@@ -34,43 +39,73 @@ public class FileSystemBackedStorageService extends S3StorageService {
 
     }
 
-
-    @SneakyThrows
-    @Override
-    public void storeObject(String objectId, InputStream data) {
-        File tempFile = File.createTempFile("test", ".tmp");
-
-        IOUtils.copy(data, new FileOutputStream(tempFile));
-
-        dataMap.put(objectId, tempFile);
-    }
-
-    @SneakyThrows
-    @Override
-    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("test", ".tmp");
-
-        IOUtils.write(data, new FileOutputStream(tempFile));
-
-        dataMap.put(objectId, tempFile);
-    }
-
-    @SneakyThrows
     @Override
     public void deleteObject(String objectId) {
-        File file = dataMap.get(objectId);
-        if (file != null) {
-            file.delete();
         dataMap.remove(objectId);
     }
-    }
 
     @Override
     public boolean objectExists(String objectId) {
         return dataMap.containsKey(objectId);
     }
 
+    @Override
+    public void init() {
+
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String objectId, T any) {
+        File tempFile = File.createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+    private ObjectMapper getMapper() {
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.findAndRegisterModules();
+        return objectMapper;
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+    public List<String> listPaths() {
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths() {
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, InputStream stream) {
+        File tempFile = File.createTempFile("test", ".tmp");
+
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
     public void clearStorage() {
-        dataMap = new HashMap<>();
+        this.dataMap.forEach((k, v) -> {
+            v.delete();
+        });
+        this.dataMap.clear();
     }
 }

persistence-service-v1/pom.xml

@@ -36,7 +36,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.14.0</version>
+                <version>1.17.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>