Pull request #423: RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)

Merge in RED/redaction-service from RED-4548-rs2PORT to release/3.107.x

* commit '15676ba13ad5cb4741092c2ad72fe582b3465cc2':
  RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.143.0</persistence-service.version>
+        <persistence-service.version>1.156.20</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
@@ -104,8 +108,9 @@ public class AnalyzeService {
                         .map(SectionArea::getPage)
                         .collect(Collectors.toSet()), sectionText.getSectionAreas())));
 
-        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store text, simplified text  and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
         return AnalyzeResult.builder()
@@ -310,4 +315,16 @@ public class AnalyzeService {
         }
     }
 
+    private SimplifiedText convert(Text text) {
+
+        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        int numberOfPages = 0;
+
+        if (text != null) {
+            numberOfPages = text.getNumberOfPages();
+            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
+        }
+        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -65,7 +65,7 @@ public class ManualRedactionSurroundingTextService {
                 var resizeItty = manualRedactions.getResizeRedactions().iterator();
                 while (resizeItty.hasNext()) {
                     var manualResizeRedaction = resizeItty.next();
-                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
+                    if (manualResizeRedaction.getValue() != null && sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
                         var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                         manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                         manualResizeRedaction.setTextAfter(surroundingText.getRight());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -223,6 +223,9 @@ public class EntitySearchUtils {
 
             if (existing.getType().equals(found.getType())) {
                 existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
                 if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
                         || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){
                     existing.setEntityType(EntityType.ENTITY);
@@ -287,6 +290,9 @@ public class EntitySearchUtils {
                 }
                 var existingEntity = existingOptional.get();
                 existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
             } else {
                 existing.add(toAdd);
             }