Pull request #445: RED-4799 remove false positive entities

Merge in RED/redaction-service from RED-4799-4 to release/3.107.x

* commit '432c4b970571b2634332b9dadd88190e99ceb6e3':
  RED-4799 remove false positive entities

redaction-service-v1/pom.xml

@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.11.0</version>
+                <version>1.15.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.143.0</persistence-service.version>
+        <persistence-service.version>1.156.20</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -20,6 +20,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>com.iqser.red.commons</groupId>
+            <artifactId>jackson-commons</artifactId>
+            <version>1.0.0</version>
+        </dependency>
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>storage-commons</artifactId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
@@ -104,8 +108,9 @@ public class AnalyzeService {
                         .map(SectionArea::getPage)
                         .collect(Collectors.toSet()), sectionText.getSectionAreas())));
 
-        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store text, simplified text  and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
         return AnalyzeResult.builder()
@@ -310,4 +315,16 @@ public class AnalyzeService {
         }
     }
 
+    private SimplifiedText convert(Text text) {
+
+        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        int numberOfPages = 0;
+
+        if (text != null) {
+            numberOfPages = text.getNumberOfPages();
+            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
+        }
+        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -65,7 +65,7 @@ public class ManualRedactionSurroundingTextService {
                 var resizeItty = manualRedactions.getResizeRedactions().iterator();
                 while (resizeItty.hasNext()) {
                     var manualResizeRedaction = resizeItty.next();
-                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
+                    if (manualResizeRedaction.getValue() != null && sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
                         var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                         manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                         manualResizeRedaction.setTextAfter(surroundingText.getRight());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -65,11 +65,11 @@ public class EntitySearchUtils {
        if(asRecommendation){
            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION);
            removeFalsePositives(found, falseRecommendations);
-           found.addAll(falseRecommendations);
+           // found.addAll(falseRecommendations);
        } else {
            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_POSITIVE);
            removeFalsePositives(found, falsePositives);
-           found.addAll(falsePositives);
+           // found.addAll(falsePositives);
        }
 
        return found;
@@ -164,6 +164,15 @@ public class EntitySearchUtils {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
+                // skip cross-type false positives
+                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
+                    continue;
+                }
+                // skip cross-type false reccommendations
+                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
+                    continue;
+                }
+
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
@@ -223,6 +232,9 @@ public class EntitySearchUtils {
 
             if (existing.getType().equals(found.getType())) {
                 existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
                 if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
                         || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){
                     existing.setEntityType(EntityType.ENTITY);
@@ -287,6 +299,9 @@ public class EntitySearchUtils {
                 }
                 var existingEntity = existingOptional.get();
                 existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
             } else {
                 existing.add(toAdd);
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.storage;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -41,7 +42,10 @@ public class RedactionStorageService {
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
+        try (var baos = new ByteArrayOutputStream()){
+            objectMapper.writeValue(baos, any);
+            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), baos);
+        }
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -7,18 +7,20 @@ import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.springframework.core.io.InputStreamResource;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.InputStream;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
 
-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService implements StorageService{
 
     private final Map<String, File> dataMap = new HashMap<>();
 
     public FileSystemBackedStorageService() {
-        super(null, null);
     }
 
     @SneakyThrows
@@ -33,16 +35,69 @@ public class FileSystemBackedStorageService extends S3StorageService {
 
     }
 
+    @Override
+    public void deleteObject(String objectId) {
+        dataMap.remove(objectId);
+    }
+
+    @Override
+    public boolean objectExists(String objectId) {
+        return dataMap.containsKey(objectId);
+    }
+
+    @Override
+    public void init() {
+
+    }
+
+    public List<String> listPaths(){
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths(){
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
     @SneakyThrows
     @Override
     public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
 
-        IOUtils.write(data, new FileOutputStream(tempFile));
+        try(var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.write(data, fileOutputStream);
+        }
 
         dataMap.put(objectId, tempFile);
     }
 
+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, InputStream stream) {
+        File tempFile = File.createTempFile("test", ".tmp");
+
+        try(var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, ByteArrayOutputStream stream) {
+
+        File tempFile = File.createTempFile("test", ".tmp");
+        try(var fileOutputStream = new FileOutputStream(tempFile)){
+            stream.writeTo(fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
     public void clearStorage() {
         this.dataMap.forEach((k, v) -> {
             v.delete();