RED-4686 - storage commons update
This commit is contained in:
Timo Bejan
parent
3e32540616
commit
9be8d46f4e
@ -29,15 +29,10 @@
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.dslplatform</groupId>
|
||||
<artifactId>dsl-json-java8</artifactId>
|
||||
<version>${dsljson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.15.0</version>
|
||||
<version>1.17.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
@ -12,17 +12,14 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@ -32,10 +29,6 @@ import java.io.InputStream;
|
||||
public class RedactionStorageService {
|
||||
|
||||
private final StorageService storageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public InputStream getStoredObject(String storageId) {
|
||||
@ -54,156 +47,66 @@ public class RedactionStorageService {
|
||||
@SneakyThrows
|
||||
@Timed("redactmanager_storeObject")
|
||||
public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
|
||||
|
||||
try(var byteArrayOutputStream = serializeObject(any)){
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), byteArrayOutputStream);
|
||||
}
|
||||
storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Timed("redactmanager_serializeObject")
|
||||
private ByteArrayOutputStream serializeObject(Object any) {
|
||||
|
||||
try {
|
||||
var baos = new ByteArrayOutputStream();
|
||||
dslJson.serialize(any, baos);
|
||||
return baos;
|
||||
} catch (com.dslplatform.json.SerializationException e) {
|
||||
// Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
var baos = new ByteArrayOutputStream();
|
||||
objectMapper.writeValue(baos, any);
|
||||
dslJson.newWriter();
|
||||
return baos;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getImportedRedactions")
|
||||
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Imported redactions not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeImportedRedactions(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeImportedRedactions")
|
||||
private ImportedRedactions deserializeImportedRedactions(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not imported redactions", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getRedactionLog")
|
||||
public RedactionLog getRedactionLog(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeRedactionLog(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeRedactionLog")
|
||||
private RedactionLog deserializeRedactionLog(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert RedactionLog", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getText")
|
||||
public Text getText(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeText(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeText")
|
||||
private Text deserializeText(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert Text", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getNerEntities")
|
||||
public NerEntities getNerEntities(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("NER Entities are not available.");
|
||||
}
|
||||
|
||||
return deserializeNerEntities(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeNerEntities")
|
||||
private NerEntities deserializeNerEntities(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert NER Entities", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getSectionGrid")
|
||||
public SectionGrid getSectionGrid(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("Section Grid is not available.");
|
||||
}
|
||||
|
||||
return deserializeSectionGrid(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeSectionGrid")
|
||||
private SectionGrid deserializeSectionGrid(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(SectionGrid.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert SectionGrid", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,22 +1,25 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.S3StorageService;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
public class FileSystemBackedStorageService implements StorageService{
|
||||
public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
private final Map<String, File> dataMap = new HashMap<>();
|
||||
|
||||
@ -50,34 +53,47 @@ public class FileSystemBackedStorageService implements StorageService{
|
||||
|
||||
}
|
||||
|
||||
public List<String> listPaths(){
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> void storeJSONObject(String objectId, T any) {
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
getMapper().writeValue(new FileOutputStream(tempFile), any);
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
private ObjectMapper getMapper() {
|
||||
var objectMapper = new ObjectMapper();
|
||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
objectMapper.findAndRegisterModules();
|
||||
return objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> T readJSONObject(String objectId, Class<T> clazz) {
|
||||
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
|
||||
throw new StorageObjectDoesNotExist("Stored object not found");
|
||||
}
|
||||
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
|
||||
}
|
||||
|
||||
public List<String> listPaths() {
|
||||
return new ArrayList<>(dataMap.keySet());
|
||||
}
|
||||
|
||||
|
||||
public List<String> listFilePaths(){
|
||||
public List<String> listFilePaths() {
|
||||
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void storeObject(String objectId, byte[] data) {
|
||||
File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
|
||||
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
IOUtils.write(data, fileOutputStream);
|
||||
}
|
||||
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String objectId, InputStream stream) {
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
try (var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
IOUtils.copy(stream, fileOutputStream);
|
||||
}
|
||||
|
||||
@ -85,19 +101,6 @@ public class FileSystemBackedStorageService implements StorageService{
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String objectId, ByteArrayOutputStream stream) {
|
||||
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)){
|
||||
stream.writeTo(fileOutputStream);
|
||||
}
|
||||
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
|
||||
public void clearStorage() {
|
||||
this.dataMap.forEach((k, v) -> {
|
||||
v.delete();
|
||||
|
||||
@ -1,30 +1,36 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
@ -48,49 +54,16 @@ import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@ -628,8 +601,7 @@ public class RedactionIntegrationTest {
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
@ -921,9 +893,9 @@ public class RedactionIntegrationTest {
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
@ -1651,8 +1623,7 @@ public class RedactionIntegrationTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@ -1719,7 +1690,7 @@ public class RedactionIntegrationTest {
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
@ -1735,10 +1706,10 @@ public class RedactionIntegrationTest {
|
||||
if (entry.getValue() == null) {
|
||||
return;
|
||||
}
|
||||
if (entry.getValue().equals("David")){
|
||||
if (entry.getValue().equals("David")) {
|
||||
assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
|
||||
}
|
||||
if (entry.getValue().equals("annotation")){
|
||||
if (entry.getValue().equals("annotation")) {
|
||||
assertThat(entry.getImportedRedactionIntersections()).isEmpty();
|
||||
}
|
||||
});
|
||||
@ -1799,9 +1770,7 @@ public class RedactionIntegrationTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
|
||||
return request;
|
||||
|
||||
|
||||
@ -454,8 +454,7 @@ public class RulesTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@ -478,9 +477,7 @@ public class RulesTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
|
||||
return request;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user