RED-4510: Generated Redaction Log and Generating will not run with profile

Merge in RED/redaction-service from RED-4686 to master

* commit '24612e347171f27eb8747772b82d13c98f40344b':
  RED-4686 - storage commons update - added annotations and test-file
  RED-4686 - storage commons update

.gitignore

@@ -26,4 +26,3 @@
 **/.DS_Store
 **/classpath-data.json
 **/dependencies-and-licenses-overview.txt
-/redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/

redaction-service-v1/pom.xml

@@ -29,15 +29,10 @@
 
     <dependencyManagement>
         <dependencies>
-            <dependency>
-                <groupId>com.dslplatform</groupId>
-                <artifactId>dsl-json-java8</artifactId>
-                <version>${dsljson.version}</version>
-            </dependency>
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.15.0</version>
+                <version>1.17.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;
 
+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -11,6 +12,7 @@ import java.util.Map;
 
 @Data
 @Builder
+@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class ImportedRedactions {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -12,17 +12,14 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -32,10 +29,6 @@ import java.io.InputStream;
 public class RedactionStorageService {
 
     private final StorageService storageService;
-    private final ObjectMapper objectMapper;
-
-    private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
-
 
     @SneakyThrows
     public InputStream getStoredObject(String storageId) {
@@ -54,156 +47,66 @@ public class RedactionStorageService {
     @SneakyThrows
     @Timed("redactmanager_storeObject")
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
-
-        try(var byteArrayOutputStream = serializeObject(any)){
-            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), byteArrayOutputStream);
-        }
+        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
     }
 
-
-    @SneakyThrows
-    @Timed("redactmanager_serializeObject")
-    private ByteArrayOutputStream serializeObject(Object any) {
-
-        try {
-            var baos = new ByteArrayOutputStream();
-            dslJson.serialize(any, baos);
-            return baos;
-        } catch (com.dslplatform.json.SerializationException e) {
-            // Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
-            var baos = new ByteArrayOutputStream();
-            objectMapper.writeValue(baos, any);
-            dslJson.newWriter();
-            return baos;
-        }
-    }
-
-
     @Timed("redactmanager_getImportedRedactions")
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
         }
-
-        return deserializeImportedRedactions(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeImportedRedactions")
-    private ImportedRedactions deserializeImportedRedactions(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not imported redactions", e);
-        }
     }
 
 
     @Timed("redactmanager_getRedactionLog")
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
         }
 
-        return deserializeRedactionLog(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeRedactionLog")
-    private RedactionLog deserializeRedactionLog(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert RedactionLog", e);
-        }
     }
 
 
     @Timed("redactmanager_getText")
     public Text getText(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
         }
-
-        return deserializeText(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeText")
-    private Text deserializeText(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert Text", e);
-        }
     }
 
 
     @Timed("redactmanager_getNerEntities")
     public NerEntities getNerEntities(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("NER Entities are not available.");
         }
-
-        return deserializeNerEntities(inputStreamResource);
     }
 
 
-    @Timed("redactmanager_deserializeNerEntities")
-    private NerEntities deserializeNerEntities(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert NER Entities", e);
-        }
-    }
-
 
     @Timed("redactmanager_getSectionGrid")
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("Section Grid is not available.");
         }
-
-        return deserializeSectionGrid(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeSectionGrid")
-    private SectionGrid deserializeSectionGrid(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(SectionGrid.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert SectionGrid", e);
-        }
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,22 +1,25 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.S3StorageService;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.springframework.core.io.InputStreamResource;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
-import java.util.zip.GZIPOutputStream;
 
-public class FileSystemBackedStorageService implements StorageService{
+public class FileSystemBackedStorageService implements StorageService {
 
     private final Map<String, File> dataMap = new HashMap<>();
 
@@ -50,34 +53,47 @@ public class FileSystemBackedStorageService implements StorageService{
 
     }
 
-    public List<String> listPaths(){
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String objectId, T any) {
+        File tempFile = File.createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+    private ObjectMapper getMapper() {
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.findAndRegisterModules();
+        return objectMapper;
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+    public List<String> listPaths() {
         return new ArrayList<>(dataMap.keySet());
     }
 
 
-    public List<String> listFilePaths(){
+    public List<String> listFilePaths() {
         return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
     }
 
 
-    @SneakyThrows
-    @Override
-    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
-
-        try(var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.write(data, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
     @Override
     @SneakyThrows
     public void storeObject(String objectId, InputStream stream) {
         File tempFile = File.createTempFile("test", ".tmp");
 
-        try(var fileOutputStream = new FileOutputStream(tempFile)) {
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
             IOUtils.copy(stream, fileOutputStream);
         }
 
@@ -85,19 +101,6 @@ public class FileSystemBackedStorageService implements StorageService{
     }
 
 
-    @Override
-    @SneakyThrows
-    public void storeObject(String objectId, ByteArrayOutputStream stream) {
-
-        File tempFile = File.createTempFile("test", ".tmp");
-        try(var fileOutputStream = new FileOutputStream(tempFile)){
-            stream.writeTo(fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
     public void clearStorage() {
         this.dataMap.forEach((k, v) -> {
             v.delete();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1,30 +1,36 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -48,49 +54,16 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import com.amazonaws.services.s3.AmazonS3;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.io.*;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.*;
+import java.util.stream.Collectors;
 
-import lombok.SneakyThrows;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -628,8 +601,7 @@ public class RedactionIntegrationTest {
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
         ClassPathResource pdfFileResource = new ClassPathResource(fileName);
@@ -921,9 +893,9 @@ public class RedactionIntegrationTest {
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                        .manualRedactions(manualRedactions)
-                        .colors(colors)
-                        .types(types)
+                .manualRedactions(manualRedactions)
+                .colors(colors)
+                .types(types)
                 .dossierId(TEST_DOSSIER_ID)
                 .fileId(TEST_FILE_ID)
                 .build());
@@ -1651,8 +1623,7 @@ public class RedactionIntegrationTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 
@@ -1719,7 +1690,7 @@ public class RedactionIntegrationTest {
         ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
 
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1735,10 +1706,10 @@ public class RedactionIntegrationTest {
             if (entry.getValue() == null) {
                 return;
             }
-            if (entry.getValue().equals("David")){
+            if (entry.getValue().equals("David")) {
                 assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
             }
-            if (entry.getValue().equals("annotation")){
+            if (entry.getValue().equals("annotation")) {
                 assertThat(entry.getImportedRedactionIntersections()).isEmpty();
             }
         });
@@ -1799,9 +1770,7 @@ public class RedactionIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        var bytes = IOUtils.toByteArray(stream);
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
 
         return request;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -114,7 +113,9 @@ public class RulesTest {
     private static final String IMPORTED_REDACTION = "imported_redaction";
     private static final String PII = "PII";
     private static final String RESOURCES_PATH = "src/test/resources/";
-    private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
+
+    private static final String REDACTION_LOG_PATH = "RedactionLog/";
+    private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
     private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
     private final static String TEST_DOSSIER_ID = "123";
     private String TEST_FILE_ID = "123";
@@ -196,7 +197,7 @@ public class RulesTest {
 
     /**
      * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
-     * If the RedactionLog already exists, the generating will be skipped
+     * If the RedactionLog already exists, it will be overwritten
      * Test is ignored, because it's for manual tests.
      */
     @Ignore
@@ -204,16 +205,17 @@ public class RulesTest {
     public void generateRedactionLogForOneFile() {
 
         String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
-        generateAndSaveRedactionLog(fileName);
+        generateAndSaveRedactionLog(fileName, true);
     }
 
 
     /**
      * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
-     * If a RedactionLog already exists, the generating will be skipped
+     * If a RedactionLog already exists, the generating for this will be skipped
+     * Please commit generated files, if not the following tests will fail
      */
-    @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
+    @Ignore// TODO PSC REMOVE
     public void generateRedactionLogForAllFiles() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
@@ -222,6 +224,9 @@ public class RulesTest {
     }
 
 
+
+
+
     /**
      * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
      * If RedactionLog Json does not exist, test will fail.
@@ -238,7 +243,7 @@ public class RulesTest {
 
     /**
      * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
-     * If RedactionLogs Json does not exist, test will fail.
+     * If RedactionLog Json for one file does not exist, whole test will fail.
      */
     @IfProfileValue(name = "test-groups", value = "rules-test")
     @Test
@@ -246,14 +251,29 @@ public class RulesTest {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
+
+        System.out.println("files: " + files);
+
         files.forEach(this::analyseFileAndCompareRedactionLog);
     }
 
 
-    @SneakyThrows
-    private void generateAndSaveRedactionLog(String fileName) {
+    @Test// TODO PSC REMOVE
+    @Ignore
+    public void foo() {
+        String fileName = "oo-home/local-working-dir/3080193/RED-REDACTIONSERVICENIGHT0-JOB1/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Compounds/26 A8637C - EU AIR3 - LCP Section 10 - Ecotoxicological studies on the plant protection product - Reference list.pdf";
+        loadSavedRedactionLog(fileName);
+    }
 
-        if (doesRedactionLogExist(fileName)) {
+
+    private void generateAndSaveRedactionLog(String fileName) {
+        generateAndSaveRedactionLog(fileName, false);
+    }
+
+    @SneakyThrows
+    private void generateAndSaveRedactionLog(String fileName, boolean force) {
+
+        if (!force && doesRedactionLogExist(fileName)) {
             System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
 
         } else {
@@ -399,7 +419,7 @@ public class RulesTest {
     private boolean doesRedactionLogExist(String pdfFileName) {
 
         File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
         String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
         File file = new File(directory, fileName);
         return file.exists();
@@ -421,7 +441,7 @@ public class RulesTest {
 
         File pdfFile = new File(pdfFileName);
 
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
         File dr = new File(directory);
         boolean created = dr.mkdirs();
         if (created) {
@@ -440,10 +460,21 @@ public class RulesTest {
     @SneakyThrows
     private RedactionLog loadSavedRedactionLog(String pdfFileName) {
 
+        System.out.println("XXXXXXXXX pdfFileName: " + pdfFileName);
+
         File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = pdfFile.getParentFile().getPath();
+        if (StringUtils.contains(directory, RESOURCES_PATH) || StringUtils.contains(directory, StringUtils.replace(RESOURCES_PATH, "/", "\\"))) {
+            directory = directory + "/" + REDACTION_LOG_PATH;
+        } else {
+            directory = REDACTION_LOG_RESOURCES_PATH + directory;
+        }
         String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
         File file = new File(directory, fileName);
+
+        System.out.println("XXXXXXXXX directory: " + directory);
+        System.out.println("XXXXXXXXX fileName:  " + fileName);
+
         assertThat(file).exists();
 
         return objectMapper.readValue(file, RedactionLog.class);
@@ -454,8 +485,7 @@ public class RulesTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 
@@ -478,9 +508,7 @@ public class RulesTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        var bytes = IOUtils.toByteArray(stream);
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
 
         return request;
 
@@ -600,8 +628,14 @@ public class RulesTest {
                     getFileNames(fileNames, path);
                 } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
                     String absolutePath = path.toAbsolutePath().toString();
+                    System.out.println("absolutePath: " + absolutePath);
                     int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
+
+                    pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + 18;
+
+                    System.out.println("pos: " + pos);
                     fileNames.add(StringUtils.substring(absolutePath, pos));
+                    System.out.println("Added: " + StringUtils.substring(absolutePath, pos));
                 }
             }
         }

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product.json

@@ -0,0 +1,533 @@
+{
+  "analysisVersion": 1,
+  "analysisNumber": 0,
+  "redactionLogEntry": [
+    {
+      "id": "96c1362cb4ea87461d8f5f141df1912a",
+      "type": "PII",
+      "value": "patrick.gardinal@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 368.44
+          },
+          "width": 141.39014,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "60 51 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 295,
+      "endOffset": 324,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "74b528293158d5266ace1eaf9ee2ad80",
+      "type": "PII",
+      "value": "+41 (0) 61 323 60 51",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 169.96002
+          },
+          "width": 94.37213,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "Gardinal Telephone number: ",
+      "textAfter": " E-mail: patrick.gardinal@syngenta.com",
+      "comments": [],
+      "startOffset": 188,
+      "endOffset": 208,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "d282cd8f02e2628c14342ed52d69c517",
+      "type": "PII",
+      "value": "+41 (61) 323 6152",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 610.12
+          },
+          "width": 83.43994,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "Micheletti Telephone number: ",
+      "textAfter": " E-mail: sebastien.micheletti@syngenta.com",
+      "comments": [],
+      "startOffset": 164,
+      "endOffset": 181,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "245cfd060d4fb078bb211fabfce6d470",
+      "type": "PII",
+      "value": "patrick.gardinal@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 144.64001
+          },
+          "width": 141.39014,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "60 51 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 217,
+      "endOffset": 246,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "67dc43606b8e1489a0594e5b779fa428",
+      "type": "PII",
+      "value": "Patrick Gardinal",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 419.08
+          },
+          "width": 73.440186,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 231,
+      "endOffset": 247,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "de79428457282a0781a827b322b5732a",
+      "type": "PII",
+      "value": "sebastien.micheletti@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 584.8
+          },
+          "width": 161.04922,
+          "height": 10.98,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "323 6152 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 190,
+      "endOffset": 223,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "41340ab190d66ca6dce0ba965ce95f59",
+      "type": "PII",
+      "value": "Patrick Gardinal",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 195.28003
+          },
+          "width": 73.440186,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 153,
+      "endOffset": 169,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "02dd2efb100839e705cdcf6f3141d9d5",
+      "type": "PII",
+      "value": "Sebastien Micheletti",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 634.84
+          },
+          "width": 91.21478,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 125,
+      "endOffset": 145,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "530c4f47f0afc48432bd849a90918f41",
+      "type": "PII",
+      "value": "+41 (0) 61 323 60 51",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 393.76
+          },
+          "width": 94.37213,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "Gardinal Telephone number: ",
+      "textAfter": " E-mail: patrick.gardinal@syngenta.com",
+      "comments": [],
+      "startOffset": 266,
+      "endOffset": 286,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    }
+  ],
+  "legalBasis": [],
+  "dictionaryVersion": 0,
+  "dossierDictionaryVersion": 0,
+  "rulesVersion": 0,
+  "legalBasisVersion": 0
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Cyprodinil/39 Cyprodinil - EU AIR3 - Document F - Notification submitted to the Commission.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":639.64},"width":55.039078,"height":11.52,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":667.12},"width":60.71997,"height":12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/270Rotated.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"36a5e8909a18a5080dd423afb8efd7a7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":159.0,"y":320.0},"width":10.208038,"height":-21.999977,"page":1}],"sectionNumber":3,"textBefore":"Product Metabolism and ","textAfter":" 1 of","comments":[],"startOffset":368,"endOffset":372,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dca119c3aadd348bc43c0b700c0075c3","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.0,"y":81.0},"width":10.208031,"height":-22.0,"page":1}],"sectionNumber":3,"textBefore":"Pages RAM 465/02 ","textAfter":" 1 Tel:","comments":[],"startOffset":399,"endOffset":403,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ef6e0f0ed413dbd7299d992d9a50840b","type":"CBI_author","value":"Crook","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":374.0,"y":389.0},"width":10.515198,"height":-27.02501,"page":1}],"sectionNumber":5,"textBefore":"Author : SJ ","textAfter":" Analytical Science","comments":[],"startOffset":12,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0fac36714dd9d1c745565c6dfe06b9b7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":514.14215,"y":47.94165},"width":24.39972,"height":11.591161,"page":2}],"sectionNumber":4,"textBefore":"RAM 465/02 ","textAfter":" 2 Summary","comments":[],"startOffset":11,"endOffset":15,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/Complex Table.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/Direction90Rotation0.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"71770acab1fa63f46c026966890b5231","type":"vertebrate","value":"rabbit","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":77.73999,"y":537.62},"width":-11.043678,"height":36.630165,"page":1}],"sectionNumber":4,"textBefore":null,"textAfter":null,"comments":[],"startOffset":41,"endOffset":47,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398329.875800000}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":true,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/US Syngenta.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"64eb5ff4ea1db8665fb6e57c5369b411","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Text in table","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":154.48782,"y":654.18},"width":9.992157,"height":19.882175,"page":1}],"sectionNumber":8,"textBefore":null,"textAfter":" 1 of","comments":[],"startOffset":54,"endOffset":58,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398731.072100800}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/foo.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"9f6837120752b85c20e27d4cb199bd51","type":"CBI_author","value":"B. Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":712.96},"width":31.589584,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":null,"textAfter":" F. Bar","comments":[],"startOffset":0,"endOffset":6,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fb930b7f3acf65a74b0b59fee32cc6fa","type":"CBI_author","value":"F. Bar","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":125.92424,"y":712.96},"width":29.137009,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"B. Foo ","textAfter":" Foo, B.","comments":[],"startOffset":7,"endOffset":13,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"435224747b87ed87b60ab363e6610fba","type":"CBI_author","value":"Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":686.8},"width":18.338272,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"Foo F. Bar ","textAfter":", B.","comments":[],"startOffset":14,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Minimal Examples/merge_images.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}