Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-4686 - storage commons update

Browse Source
This commit is contained in:
Timo Bejan 2022-07-25 22:19:23 +03:00
parent eeb162ce61
commit a876ace06b
11 changed files with 129 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
persistence-service-v1/persistence-service-server-v1/pom.xml
View File

@ -186,6 +186,19 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessors>
<annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
<annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
</annotationProcessors>
</configuration>
</plugin>
<plugin>
<!-- generate git.properties for exposure in /info -->
<groupId>pl.project13.maven</groupId>

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -120,10 +120,8 @@ public class ReanalysisController implements ReanalysisResource {
public void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest) {
var importedRedactions = fileManagementStorageService.getImportedRedactions(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId());
importedRedactions.getImportedRedactions().entrySet().forEach(entry -> {
entry.getValue().removeIf(v -> deleteImportedRedactionsRequest.getAnnotationIds().contains(v.getId()));
});
fileManagementStorageService.storeObject(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), FileType.IMPORTED_REDACTIONS, objectMapper.writeValueAsBytes(importedRedactions));
importedRedactions.getImportedRedactions().forEach((key, value) -> value.removeIf(v -> deleteImportedRedactionsRequest.getAnnotationIds().contains(v.getId())));
fileManagementStorageService.storeJSONObject(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), FileType.IMPORTED_REDACTIONS, importedRedactions);
fileStatusService.setStatusFullReprocess(deleteImportedRedactionsRequest.getDossierId(), deleteImportedRedactionsRequest.getFileId(), true, false);
}

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReportTemplateController.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.controller;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
@ -55,7 +56,7 @@ public class ReportTemplateController implements ReportTemplateResource {
String storageId = StorageIdUtils.getReportStorageId(reportTemplateUploadRequest.getDossierTemplateId(), reportTemplateUploadRequest
.getFileName());
storageService.storeObject(storageId, reportTemplateUploadRequest.getTemplate());
storageService.storeObject(storageId, new ByteArrayInputStream(reportTemplateUploadRequest.getTemplate()));
if (templateId != null) {
reportTemplatePersistenceService.updateTemplate(reportTemplateUploadRequest.getDossierTemplateId(), templateId, ReportTemplateUpdateRequest

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java
View File

@ -75,12 +75,10 @@ public class DictionaryToEntityMigration2 extends Migration {
if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
}
} catch (JsonProcessingException e) {
throw new RuntimeException("Migration failed");
} catch (NotFoundException e) {
} catch (Exception e) {
log.info("redactionLog {} does not exsist", file.getId());
}
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
@ -12,6 +11,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
@Slf4j
@Setter
@Service
@ -60,7 +61,7 @@ public class MigrateHighlights3 extends Migration {
}
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
new ByteArrayInputStream(fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)));
}

41
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java
View File

@ -9,14 +9,13 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.SectionGrid;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import liquibase.pro.packaged.T;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
@Slf4j
@ -24,7 +23,6 @@ import java.io.InputStream;
@RequiredArgsConstructor
public class FileManagementStorageService {
private final ObjectMapper objectMapper;
private final StorageService storageService;
@ -42,10 +40,14 @@ public class FileManagementStorageService {
return IOUtils.toByteArray(storageService.getObject(storageId).getInputStream());
}
public void storeObject(String dossierId, String fileId, FileType fileType, InputStream stream){
public void storeObject(String dossierId, String fileId, FileType fileType, byte[] data) {
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), stream);
}
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), data);
public <T> void storeJSONObject(String dossierId, String fileId, FileType fileType, T jsonObject){
storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), jsonObject);
}
@ -56,45 +58,34 @@ public class FileManagementStorageService {
public RedactionLog getRedactionLog(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("Text not available.");
throw new NotFoundException("RedactionLog does not exist");
}
try {
return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class);
} catch (IOException e) {
throw new RuntimeException("Could not convert Text", e);
}
}
public SectionGrid getSectionGrid(String dossierId, String fileId) {
var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
try {
return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
} catch (IOException e) {
throw new RuntimeException("Could not convert RedactionLog", e);
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("SectionGrid not available.");
throw new NotFoundException("RedactionLog does not exist");
} catch (Exception e) {
throw new RuntimeException("Could not convert SectionGrid", e);
}
}
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
} catch (StorageObjectDoesNotExist e) {
throw new NotFoundException("ImportedRedactions does not exist");
}
try {
return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
} catch (IOException e) {
} catch (Exception e) {
throw new RuntimeException("Could not convert ImportedRedactions", e);
}
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ImageMessageReceiver.java
View File

@ -1,21 +1,21 @@
package com.iqser.red.service.peristence.v1.server.service;
import java.io.IOException;
import java.util.HashMap;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.HashMap;
@Slf4j
@Service
@ -33,14 +33,13 @@ public class ImageMessageReceiver {
@RabbitListener(queues = MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE)
public void receive(byte[] in) {
HashMap<String, Object> imageResponse = objectMapper.readValue(in, new TypeReference<>() {
});
JsonNode imageResponse = objectMapper.readTree(in);
String dossierId = (String) imageResponse.get("dossierId");
String fileId = (String) imageResponse.get("fileId");
String dossierId = imageResponse.path("dossierId").asText();
String fileId = imageResponse.path("fileId").asText();
if(settings.isStoreImageFile()) {
fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, in);
if (settings.isStoreImageFile()) {
fileManagementStorageService.storeObject(dossierId, fileId, FileType.IMAGE_INFO, new ByteArrayInputStream(in));
}
fileStatusService.setStatusAnalyse(dossierId, fileId, false);

11
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -18,6 +18,7 @@ import org.apache.commons.codec.binary.Base64;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
@ -56,11 +57,11 @@ public class FileTesterAndProvider {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, 1,
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, new RedactionLog(1, 1,
List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
null, 0, 0, 0, 0)));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid()));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, objectMapper.writeValueAsBytes("bytes of the file"));
null, 0, 0, 0, 0));
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, new SectionGrid());
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, new ByteArrayInputStream(objectMapper.writeValueAsBytes("bytes of the file")));
return file;
}
@ -71,7 +72,7 @@ public class FileTesterAndProvider {
var fileId = Base64.encodeBase64String((dossier.getId() + fileName).getBytes(StandardCharsets.UTF_8));
AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1");
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
JSONPrimitive<String> uploadResult = uploadClient.upload(upload, false);
return uploadResult.getValue();

10
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -169,9 +169,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII");
// assume file is already proccessed once, test that add to dict triggers reanalysis
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.TEXT, "{}");
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "{}");
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "{}");
fileStatusPersistenceService.updateProcessingStatus(file.getId(), ProcessingStatus.PROCESSED);
var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
@ -308,7 +308,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.type("manual")
.value("value entry")
.build()), null, 0, 0, 0, 0);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(redactionLog);
manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(), UpdateRedactionRequest.builder()
@ -334,7 +334,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedRemoveRedaction.getStatus()).isEqualTo(AnnotationStatus.DECLINED);
var redLog = new RedactionLog(1, 1, List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()), null, 0, 0, 0, 0);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redLog));
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redLog);
when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(redLog);
var removeRedaction2 = manualRedactionClient.addRemoveRedaction(dossier.getId(), file.getId(), List.of(RemoveRedactionRequest.builder()

97
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java
View File

@ -1,5 +1,8 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.S3StorageService;
import com.iqser.red.storage.commons.service.StorageService;
@ -11,15 +14,17 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class FileSystemBackedStorageService extends S3StorageService {
public class FileSystemBackedStorageService implements StorageService {
private Map<String, File> dataMap = new HashMap<>();
private final Map<String, File> dataMap = new HashMap<>();
public FileSystemBackedStorageService() {
super(null, null);
}
@SneakyThrows
@ -34,35 +39,9 @@ public class FileSystemBackedStorageService extends S3StorageService {
}
@SneakyThrows
@Override
public void storeObject(String objectId, InputStream data) {
File tempFile = File.createTempFile("test", ".tmp");
IOUtils.copy(data, new FileOutputStream(tempFile));
dataMap.put(objectId, tempFile);
}
@SneakyThrows
@Override
public void storeObject(String objectId, byte[] data) {
File tempFile = File.createTempFile("test", ".tmp");
IOUtils.write(data, new FileOutputStream(tempFile));
dataMap.put(objectId, tempFile);
}
@SneakyThrows
@Override
public void deleteObject(String objectId) {
File file = dataMap.get(objectId);
if (file != null) {
file.delete();
dataMap.remove(objectId);
}
dataMap.remove(objectId);
}
@Override
@ -70,7 +49,63 @@ public class FileSystemBackedStorageService extends S3StorageService {
return dataMap.containsKey(objectId);
}
@Override
public void init() {
}
@Override
@SneakyThrows
public <T> void storeJSONObject(String objectId, T any) {
File tempFile = File.createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
private ObjectMapper getMapper() {
var objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.registerModule(new JavaTimeModule());
objectMapper.findAndRegisterModules();
return objectMapper;
}
@Override
@SneakyThrows
public <T> T readJSONObject(String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
}
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
}
public List<String> listPaths() {
return new ArrayList<>(dataMap.keySet());
}
public List<String> listFilePaths() {
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
}
@Override
@SneakyThrows
public void storeObject(String objectId, InputStream stream) {
File tempFile = File.createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
public void clearStorage() {
dataMap = new HashMap<>();
this.dataMap.forEach((k, v) -> {
v.delete();
});
this.dataMap.clear();
}
}

2
persistence-service-v1/pom.xml
View File

@ -36,7 +36,7 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>1.14.0</version>
<version>1.17.0</version>
<scope>import</scope>
<type>pom</type>
</dependency>