DEBUG log

redaction-service-image-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.iqser.red</groupId>
         <artifactId>platform-docker-dependency</artifactId>
-        <version>1.1.0</version>
+        <version>1.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.1.5</version>
+        <version>1.1.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.7.0</version>
+                <version>1.4.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -20,7 +20,7 @@
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>persistence-service-api-v1</artifactId>
-            <version>0.44.0</version>
+            <version>0.22.0</version>
         </dependency>
     </dependencies>
 </project>

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -24,12 +24,12 @@
         <dependency>
             <groupId>org.drools</groupId>
             <artifactId>drools-core</artifactId>
-            <version>7.59.0.Final</version>
+            <version>7.37.0.Final</version>
         </dependency>
         <dependency>
             <groupId>org.kie</groupId>
             <artifactId>kie-spring</artifactId>
-            <version>7.59.0.Final</version>
+            <version>7.37.0.Final</version>
         </dependency>
         <dependency>
             <groupId>org.locationtech.jts</groupId>
@@ -101,6 +101,12 @@
             <artifactId>test-commons</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.amqp</groupId>
+            <artifactId>spring-rabbit-test</artifactId>
+            <version>2.3.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import com.amazonaws.services.kms.model.NotFoundException;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
@@ -153,10 +152,6 @@ public class RedactionController implements RedactionResource {
 
         var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
 
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
         log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
         if (redactionLog.getAnalysisVersion() == 0) {
             // old redaction logs are returned directly

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -35,11 +35,12 @@ public class RedactionMessageReceiver {
     public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
 
         var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
-        log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
+        log.info("Processing analyze request: {}", analyzeRequest);
         AnalyzeResult result;
         if (analyzeRequest.isReanalyseOnlyIfPossible()) {
             result = analyzeService.reanalyze(analyzeRequest);
-            log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+            log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest
+                    .getFileId(), result.getDuration());
         } else {
             // TODO Seperate stucture analysis by other queue
             analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -52,9 +52,11 @@ public class DictionaryService {
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
+        log.info("From version is: {} and version is {}", fromVersion, version);
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
         List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
+        log.info("Entries" + dictionaryModels.stream().filter(dm -> dm.getType().equals("PII")).findFirst().get().getEntries());
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -62,6 +64,7 @@ public class DictionaryService {
                 }
             });
         });
+        log.info("New Values are: " + newValues);
 
         if(dictionariesByDossier.containsKey(dossierId)) {
             dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/NerAnalyserService.java

@@ -1,17 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import org.apache.commons.codec.binary.Base64;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.codec.binary.Base64;
-import org.springframework.stereotype.Service;
-
-import java.util.stream.Collectors;
 
 @Slf4j
 @Service
@@ -29,24 +31,20 @@ public class NerAnalyserService {
 
             long start = System.currentTimeMillis();
 
-            if (text != null) {
-                var nerRequest = EntityRecognitionRequest.builder()
-                        .data(text.getSectionTexts()
-                                .stream()
-                                .map(sectionText -> new EntityRecognitionSection(sectionText.getSectionNumber(), new String(Base64
-                                        .encodeBase64(sectionText
-                                                .getText().getBytes()))))
-                                .collect(Collectors.toList()))
-                        .build();
+            var nerRequest = EntityRecognitionRequest.builder()
+                    .data(text.getSectionTexts()
+                            .stream()
+                            .map(sectionText -> new EntityRecognitionSection(sectionText.getSectionNumber(), new String(Base64
+                                    .encodeBase64(sectionText
+                                    .getText().getBytes()))))
+                            .collect(Collectors.toList()))
+                    .build();
 
-                var nerResponse = entityRecognitionClient.findAuthors(nerRequest);
+            var nerResponse = entityRecognitionClient.findAuthors(nerRequest);
 
-                log.info("Computing NER entities took: {} ms for dossierId {} and fileId {}", System.currentTimeMillis() - start, dossierId, fileId);
+            log.info("Computing NER entities took: {} ms for dossierId {} and fileId {}", System.currentTimeMillis() - start, dossierId, fileId);
 
-                redactionStorageService.storeObject(dossierId, fileId, FileType.NER_ENTITIES, nerResponse);
-            } else {
-                log.warn("Warning, text for file: {} in dossier: {} is null", fileId, dossierId);
-            }
+            redactionStorageService.storeObject(dossierId, fileId, FileType.NER_ENTITIES, nerResponse);
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -34,7 +34,7 @@ public class RedactionLogMergeService {
     public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId,
                                               ManualRedactions manualRedactions, Set<Integer> excludedPages) {
 
-        log.info("Merging Redaction log with manual redactions");
+        log.info("Merging Redaction log with manual redactions {}", manualRedactions);
         if (manualRedactions != null) {
 
             var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -13,7 +13,6 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -65,19 +64,8 @@ public class PdfSegmentationService {
         PDDocument pdDocument = null;
         try {
             //create tempFile
-            File tempFile;
-
-            if(SystemUtils.IS_OS_UNIX) {
-                FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
-                tempFile = Files.createTempFile("document", ".pdf", attr).toFile();
-            }
-            else {
-                tempFile = Files.createTempFile("document", ".pdf").toFile();
-                tempFile.setReadable(true, true);
-                tempFile.setWritable(true, true);
-                tempFile.setExecutable(true, true);
-            }
-
+            FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
+            File tempFile = Files.createTempFile("document", ".pdf", attr).toFile();
             try (var fos = new FileOutputStream(tempFile)) {
                 IOUtils.copy(documentInputStream, fos);
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -40,4 +40,3 @@ storage:
   bucket-name: 'redaction'
   region: 'us-east-1'
   endpoint: 'https://s3.amazonaws.com'
-  backend: 's3'

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.S3StorageService;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
@@ -13,7 +12,7 @@ import java.io.FileOutputStream;
 import java.util.HashMap;
 import java.util.Map;
 
-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService extends StorageService {
 
     private final Map<String, File> dataMap = new HashMap<>();
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -21,11 +21,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeServic
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -139,7 +137,7 @@ public class RedactionIntegrationTest {
     private final static String TEST_FILE_ID = "123";
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -576,7 +574,7 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated3.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated3.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long rstart = System.currentTimeMillis();
@@ -688,7 +686,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("first analysis duration: " + (end - start));
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
             fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
         }
 
@@ -760,7 +758,7 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
@@ -796,7 +794,7 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long end = System.currentTimeMillis();
@@ -892,7 +890,7 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long end = System.currentTimeMillis();
@@ -918,7 +916,7 @@ public class RedactionIntegrationTest {
 
         RedactionResult result = redactionController.classify(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Classified.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Classified.pdf")) {
             fileOutputStream.write(result.getDocument());
         }
     }
@@ -940,7 +938,7 @@ public class RedactionIntegrationTest {
 
         RedactionResult result = redactionController.sections(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Sections.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Sections.pdf")) {
             fileOutputStream.write(result.getDocument());
         }
     }
@@ -962,7 +960,7 @@ public class RedactionIntegrationTest {
 
         RedactionResult result = redactionController.htmlTables(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Tables.html")) {
             fileOutputStream.write(result.getDocument());
         }
     }
@@ -984,7 +982,7 @@ public class RedactionIntegrationTest {
 
         RedactionResult result = redactionController.htmlTables(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Tables.html")) {
             fileOutputStream.write(result.getDocument());
         }
     }
@@ -1054,7 +1052,7 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long end = System.currentTimeMillis();
@@ -1082,12 +1080,4 @@ public class RedactionIntegrationTest {
         }
     }
 
-    private static String getTemporaryDirectory() {
-        String tmpdir = System.getProperty("java.io.tmpdir");
-        if (StringUtils.isNotBlank(tmpdir)) {
-            return tmpdir;
-        }
-        return "/tmp";
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -13,7 +13,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingC
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.platform.commons.util.StringUtils;
 import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
@@ -96,7 +95,7 @@ public class PdfSegmentationServiceTest {
             for (PdfImage image : page.getImages()) {
                 try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
                     ImageIO.write(image.getImage(), "png", baos);
-                    try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Image " + i + ".png")) {
+                    try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Image " + i + ".png")) {
                         fileOutputStream.write(baos.toByteArray());
                     }
                 }
@@ -240,12 +239,4 @@ public class PdfSegmentationServiceTest {
                         .equals(firstTableHeaderCells))).isTrue();
     }
 
-    private static String getTemporaryDirectory() {
-        String tmpdir = System.getProperty("java.io.tmpdir");
-        if (StringUtils.isNotBlank(tmpdir)) {
-            return tmpdir;
-        }
-        return "/tmp";
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml

@@ -18,10 +18,3 @@ platform.multi-tenancy:
 redaction-service:
   enable-image-classification: false
   enable-entity-recognition: false
-
-storage:
-  signer-type: 'AWSS3V4SignerType'
-  bucket-name: 'redaction'
-  region: 'us-east-1'
-  endpoint: 'https://s3.amazonaws.com'
-  backend: 's3'