Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

fixed in-memory storage issues

Browse Source
This commit is contained in:
Timo 2021-04-17 09:10:40 +03:00
parent 8060e3a29f
commit 169ab20351
4 changed files with 64 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FilySystemBackedStorageService.java Normal file
View File

@ -0,0 +1,51 @@
package com.iqser.red.service.redaction.v1.server;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.Map;
public class FilySystemBackedStorageService extends StorageService {
private Map<String, File> dataMap = new HashMap<>();
public FilySystemBackedStorageService() {
super(null, null);
}
@SneakyThrows
@Override
public InputStreamResource getObject(String objectId) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
return new InputStreamResource(new FileInputStream(res));
}
@SneakyThrows
@Override
public void storeObject(String objectId, byte[] data) {
File tempFile = File.createTempFile("test", ".tmp");
IOUtils.write(data, new FileOutputStream(tempFile));
dataMap.put(objectId, tempFile);
}
public void clearStorage() {
this.dataMap.forEach((k, v) -> {
v.delete();
});
this.dataMap.clear();
}
}

34
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/InMemoryStorageService.java
View File

@ -1,34 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import org.springframework.core.io.InputStreamResource;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;
public class InMemoryStorageService extends StorageService {
private Map<String, byte[]> dataMap = new HashMap<>();
public InMemoryStorageService() {
super(null, null);
}
@Override
public InputStreamResource getObject(String objectId) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
return new InputStreamResource(new ByteArrayInputStream(res));
}
@Override
public void storeObject(String objectId, byte[] data) {
dataMap.put(objectId, data);
}
}

13
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@ -134,12 +135,20 @@ public class RedactionIntegrationTest {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new InMemoryStorageService();
return new FilySystemBackedStorageService();
}
}
@After
public void cleanupStorage() {
if (this.storageService instanceof FilySystemBackedStorageService) {
((FilySystemBackedStorageService) this.storageService).clearStorage();
}
}
@Before
public void stubClients() {
@ -444,7 +453,7 @@ public class RedactionIntegrationTest {
@Test
@Ignore
public void testLargeScannedFileOOM(){
public void testLargeScannedFileOOM() {
AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
MemoryStats.printMemoryStats();
AnalyzeResult result = redactionController.analyze(request);

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java
View File

@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.amazonaws.services.s3.AmazonS3;
import com.iqser.red.service.configuration.v1.api.model.*;
import com.iqser.red.service.redaction.v1.server.InMemoryStorageService;
import com.iqser.red.service.redaction.v1.server.FilySystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@ -97,7 +97,7 @@ public class EntityRedactionServiceTest {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new InMemoryStorageService();
return new FilySystemBackedStorageService();
}
}