Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #432: RED-4686 - storage commons update

Browse Source 
Merge in RED/redaction-service from RED-4686 to master

* commit '24612e347171f27eb8747772b82d13c98f40344b':
  RED-4686 - storage commons update - added annotations and test-file
  RED-4686 - storage commons update
This commit is contained in:
Timo Bejan 2022-07-25 21:19:43 +02:00
commit 9c8e2f047e
7 changed files with 96 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
redaction-service-v1/pom.xml
View File

@ -29,15 +29,10 @@
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
<dependency>
<groupId>com.dslplatform</groupId>
<artifactId>dsl-json-java8</artifactId>
<version>${dsljson.version}</version>
</dependency>
<dependency> <dependency>
<groupId>com.iqser.red</groupId> <groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId> <artifactId>platform-commons-dependency</artifactId>
<version>1.15.0</version> <version>1.17.0</version>
<scope>import</scope> <scope>import</scope>
<type>pom</type> <type>pom</type>
</dependency> </dependency>

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.model; package com.iqser.red.service.redaction.v1.model;
import com.dslplatform.json.CompiledJson;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@ -11,6 +12,7 @@ import java.util.Map;
@Data @Data
@Builder @Builder
@CompiledJson
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor @AllArgsConstructor
public class ImportedRedactions { public class ImportedRedactions {

109
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -12,17 +12,14 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.Getter; import lombok.Getter;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -32,10 +29,6 @@ import java.io.InputStream;
public class RedactionStorageService { public class RedactionStorageService {
private final StorageService storageService; private final StorageService storageService;
private final ObjectMapper objectMapper;
private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
@SneakyThrows @SneakyThrows
public InputStream getStoredObject(String storageId) { public InputStream getStoredObject(String storageId) {
@ -54,156 +47,66 @@ public class RedactionStorageService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_storeObject") @Timed("redactmanager_storeObject")
public void storeObject(String dossierId, String fileId, FileType fileType, Object any) { public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
try(var byteArrayOutputStream = serializeObject(any)){
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), byteArrayOutputStream);
}
} }
@SneakyThrows
@Timed("redactmanager_serializeObject")
private ByteArrayOutputStream serializeObject(Object any) {
try {
var baos = new ByteArrayOutputStream();
dslJson.serialize(any, baos);
return baos;
} catch (com.dslplatform.json.SerializationException e) {
// Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
var baos = new ByteArrayOutputStream();
objectMapper.writeValue(baos, any);
dslJson.newWriter();
return baos;
}
}
@Timed("redactmanager_getImportedRedactions") @Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS)); return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available."); log.debug("Imported redactions not available.");
return null; return null;
} }
return deserializeImportedRedactions(inputStreamResource);
}
@Timed("redactmanager_deserializeImportedRedactions")
private ImportedRedactions deserializeImportedRedactions(InputStreamResource inputStreamResource) {
try {
return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
} catch (IOException e) {
throw new RuntimeException("Could not imported redactions", e);
}
} }
@Timed("redactmanager_getRedactionLog") @Timed("redactmanager_getRedactionLog")
public RedactionLog getRedactionLog(String dossierId, String fileId) { public RedactionLog getRedactionLog(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG)); return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Text not available."); log.debug("Text not available.");
return null; return null;
} }
return deserializeRedactionLog(inputStreamResource);
}
@Timed("redactmanager_deserializeRedactionLog")
private RedactionLog deserializeRedactionLog(InputStreamResource inputStreamResource) {
try {
return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
} catch (IOException e) {
throw new RuntimeException("Could not convert RedactionLog", e);
}
} }
@Timed("redactmanager_getText") @Timed("redactmanager_getText")
public Text getText(String dossierId, String fileId) { public Text getText(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Text not available."); log.debug("Text not available.");
return null; return null;
} }
return deserializeText(inputStreamResource);
}
@Timed("redactmanager_deserializeText")
private Text deserializeText(InputStreamResource inputStreamResource) {
try {
return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
} catch (IOException e) {
throw new RuntimeException("Could not convert Text", e);
}
} }
@Timed("redactmanager_getNerEntities") @Timed("redactmanager_getNerEntities")
public NerEntities getNerEntities(String dossierId, String fileId) { public NerEntities getNerEntities(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
throw new NotFoundException("NER Entities are not available."); throw new NotFoundException("NER Entities are not available.");
} }
return deserializeNerEntities(inputStreamResource);
} }
@Timed("redactmanager_deserializeNerEntities")
private NerEntities deserializeNerEntities(InputStreamResource inputStreamResource) {
try {
return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
} catch (IOException e) {
throw new RuntimeException("Could not convert NER Entities", e);
}
}
@Timed("redactmanager_getSectionGrid") @Timed("redactmanager_getSectionGrid")
public SectionGrid getSectionGrid(String dossierId, String fileId) { public SectionGrid getSectionGrid(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try { try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)); return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
throw new NotFoundException("Section Grid is not available."); throw new NotFoundException("Section Grid is not available.");
} }
return deserializeSectionGrid(inputStreamResource);
}
@Timed("redactmanager_deserializeSectionGrid")
private SectionGrid deserializeSectionGrid(InputStreamResource inputStreamResource) {
try {
return dslJson.deserialize(SectionGrid.class, inputStreamResource.getInputStream());
} catch (IOException e) {
throw new RuntimeException("Could not convert SectionGrid", e);
}
} }

69
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java
View File

@ -1,22 +1,25 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.S3StorageService;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.InputStream; import java.io.InputStream;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.zip.GZIPOutputStream;
public class FileSystemBackedStorageService implements StorageService{ public class FileSystemBackedStorageService implements StorageService {
private final Map<String, File> dataMap = new HashMap<>(); private final Map<String, File> dataMap = new HashMap<>();
@ -50,34 +53,47 @@ public class FileSystemBackedStorageService implements StorageService{
} }
public List<String> listPaths(){ @Override
@SneakyThrows
public <T> void storeJSONObject(String objectId, T any) {
File tempFile = File.createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
private ObjectMapper getMapper() {
var objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.registerModule(new JavaTimeModule());
objectMapper.findAndRegisterModules();
return objectMapper;
}
@Override
@SneakyThrows
public <T> T readJSONObject(String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
}
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
}
public List<String> listPaths() {
return new ArrayList<>(dataMap.keySet()); return new ArrayList<>(dataMap.keySet());
} }
public List<String> listFilePaths(){ public List<String> listFilePaths() {
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList()); return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
} }
@SneakyThrows
@Override
public void storeObject(String objectId, byte[] data) {
File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
try(var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.write(data, fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
@Override @Override
@SneakyThrows @SneakyThrows
public void storeObject(String objectId, InputStream stream) { public void storeObject(String objectId, InputStream stream) {
File tempFile = File.createTempFile("test", ".tmp"); File tempFile = File.createTempFile("test", ".tmp");
try(var fileOutputStream = new FileOutputStream(tempFile)) { try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream); IOUtils.copy(stream, fileOutputStream);
} }
@ -85,19 +101,6 @@ public class FileSystemBackedStorageService implements StorageService{
} }
@Override
@SneakyThrows
public void storeObject(String objectId, ByteArrayOutputStream stream) {
File tempFile = File.createTempFile("test", ".tmp");
try(var fileOutputStream = new FileOutputStream(tempFile)){
stream.writeTo(fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
public void clearStorage() { public void clearStorage() {
this.dataMap.forEach((k, v) -> { this.dataMap.forEach((k, v) -> {
v.delete(); v.delete();

129
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,30 +1,36 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat; import com.amazonaws.services.s3.AmazonS3;
import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedReader; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import java.io.ByteArrayInputStream; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import java.io.File; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import java.io.FileInputStream; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import java.io.FileOutputStream; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
import java.io.IOException; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import java.io.InputStream; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import java.io.InputStreamReader; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import java.net.URL; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import java.time.OffsetDateTime; import com.iqser.red.service.redaction.v1.model.*;
import java.time.ZoneOffset; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import java.util.ArrayList; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import java.util.HashMap; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import java.util.HashSet; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import java.util.List; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import java.util.Map; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import java.util.Set; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import java.util.UUID; import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import java.util.stream.Collectors; import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
import org.apache.commons.io.IOUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore; import org.junit.Ignore;
@ -48,49 +54,16 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3; import java.io.*;
import com.fasterxml.jackson.core.type.TypeReference; import java.net.URL;
import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.charset.StandardCharsets;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import java.time.ZoneOffset;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import java.util.*;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -628,8 +601,7 @@ public class RedactionIntegrationTest {
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json"); ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
var bytes = IOUtils.toByteArray(responseJson.getInputStream()); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource(fileName); ClassPathResource pdfFileResource = new ClassPathResource(fileName);
@ -921,9 +893,9 @@ public class RedactionIntegrationTest {
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions) .manualRedactions(manualRedactions)
.colors(colors) .colors(colors)
.types(types) .types(types)
.dossierId(TEST_DOSSIER_ID) .dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
@ -1651,8 +1623,7 @@ public class RedactionIntegrationTest {
private void loadNerForTest() { private void loadNerForTest() {
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
var bytes = IOUtils.toByteArray(responseJson.getInputStream()); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
} }
@ -1719,7 +1690,7 @@ public class RedactionIntegrationTest {
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json"); ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream())); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
@ -1735,10 +1706,10 @@ public class RedactionIntegrationTest {
if (entry.getValue() == null) { if (entry.getValue() == null) {
return; return;
} }
if (entry.getValue().equals("David")){ if (entry.getValue().equals("David")) {
assertThat(entry.getImportedRedactionIntersections()).hasSize(1); assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
} }
if (entry.getValue().equals("annotation")){ if (entry.getValue().equals("annotation")) {
assertThat(entry.getImportedRedactionIntersections()).isEmpty(); assertThat(entry.getImportedRedactionIntersections()).isEmpty();
} }
}); });
@ -1799,9 +1770,7 @@ public class RedactionIntegrationTest {
.lastProcessed(OffsetDateTime.now()) .lastProcessed(OffsetDateTime.now())
.build(); .build();
var bytes = IOUtils.toByteArray(stream); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
return request; return request;

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -454,8 +454,7 @@ public class RulesTest {
private void loadNerForTest() { private void loadNerForTest() {
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
var bytes = IOUtils.toByteArray(responseJson.getInputStream()); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
} }
@ -478,9 +477,7 @@ public class RulesTest {
.lastProcessed(OffsetDateTime.now()) .lastProcessed(OffsetDateTime.now())
.build(); .build();
var bytes = IOUtils.toByteArray(stream); storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
return request; return request;

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/data/test-file-2.pdf Normal file
View File

Binary file not shown.