From 169ab20351ee97a34c42b9b8e7e7bf4ff86a5c4f Mon Sep 17 00:00:00 2001 From: Timo Date: Sat, 17 Apr 2021 09:10:40 +0300 Subject: [PATCH] fixed in-memory storage issues --- .../FilySystemBackedStorageService.java | 51 +++++++++++++++++++ .../v1/server/InMemoryStorageService.java | 34 ------------- .../v1/server/RedactionIntegrationTest.java | 13 ++++- .../service/EntityRedactionServiceTest.java | 4 +- 4 files changed, 64 insertions(+), 38 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FilySystemBackedStorageService.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/InMemoryStorageService.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FilySystemBackedStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FilySystemBackedStorageService.java new file mode 100644 index 00000000..cff5698f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FilySystemBackedStorageService.java @@ -0,0 +1,51 @@ +package com.iqser.red.service.redaction.v1.server; + +import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; +import com.iqser.red.storage.commons.service.StorageService; +import lombok.SneakyThrows; +import org.apache.commons.io.IOUtils; +import org.springframework.core.io.InputStreamResource; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map; + +public class FilySystemBackedStorageService extends StorageService { + + private Map dataMap = new HashMap<>(); + + public FilySystemBackedStorageService() { + super(null, null); + } + + @SneakyThrows + @Override + public InputStreamResource getObject(String objectId) { + + var res = dataMap.get(objectId); + if (res == null) { + throw new StorageObjectDoesNotExist(new RuntimeException()); + } + return new InputStreamResource(new FileInputStream(res)); + + } + + @SneakyThrows + @Override + public void storeObject(String objectId, byte[] data) { + File tempFile = File.createTempFile("test", ".tmp"); + + IOUtils.write(data, new FileOutputStream(tempFile)); + + dataMap.put(objectId, tempFile); + } + + public void clearStorage() { + this.dataMap.forEach((k, v) -> { + v.delete(); + }); + this.dataMap.clear(); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/InMemoryStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/InMemoryStorageService.java deleted file mode 100644 index e73acb8b..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/InMemoryStorageService.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.iqser.red.service.redaction.v1.server; - -import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; -import com.iqser.red.storage.commons.service.StorageService; -import org.springframework.core.io.InputStreamResource; - -import java.io.ByteArrayInputStream; -import java.util.HashMap; -import java.util.Map; - -public class InMemoryStorageService extends StorageService { - - private Map dataMap = new HashMap<>(); - - public InMemoryStorageService() { - super(null, null); - } - - @Override - public InputStreamResource getObject(String objectId) { - - var res = dataMap.get(objectId); - if (res == null) { - throw new StorageObjectDoesNotExist(new RuntimeException()); - } - return new InputStreamResource(new ByteArrayInputStream(res)); - - } - - @Override - public void storeObject(String objectId, byte[] data) { - dataMap.put(objectId, data); - } -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 3e4dc0eb..6d84c905 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService import com.iqser.red.storage.commons.service.StorageService; import lombok.SneakyThrows; import org.apache.commons.io.IOUtils; +import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -134,12 +135,20 @@ public class RedactionIntegrationTest { @Bean @Primary public StorageService inmemoryStorage() { - return new InMemoryStorageService(); + return new FilySystemBackedStorageService(); } } + @After + public void cleanupStorage() { + if (this.storageService instanceof FilySystemBackedStorageService) { + ((FilySystemBackedStorageService) this.storageService).clearStorage(); + } + } + + @Before public void stubClients() { @@ -444,7 +453,7 @@ public class RedactionIntegrationTest { @Test @Ignore - public void testLargeScannedFileOOM(){ + public void testLargeScannedFileOOM() { AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); MemoryStats.printMemoryStats(); AnalyzeResult result = redactionController.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index 25213df4..8c19e0d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.amazonaws.services.s3.AmazonS3; import com.iqser.red.service.configuration.v1.api.model.*; -import com.iqser.red.service.redaction.v1.server.InMemoryStorageService; +import com.iqser.red.service.redaction.v1.server.FilySystemBackedStorageService; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -97,7 +97,7 @@ public class EntityRedactionServiceTest { @Bean @Primary public StorageService inmemoryStorage() { - return new InMemoryStorageService(); + return new FilySystemBackedStorageService(); } }