RED-125: Section must know its headlines

RED-156: Return RedactionLog

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Point {
+
+    private float x;
+    private float y;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Position.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Position {
+
+    private Point topLeft;
+    private Point topRight;
+    private Point bottomLeft;
+    private Point bottomRight;
+
+    private int page;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class RedactionLog {
+
+    private List<RedactionLogEntity> redactionLogEntity;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntity.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class RedactionLogEntity {
+
+    private String id;
+    private String type;
+    private String value;
+    private String reason;
+    private boolean redacted;
+    private String section;
+    private float[] color;
+    private List<Position> positions = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -13,4 +13,6 @@ public class RedactionResult {
 
     private byte[] document;
     private int numberOfPages;
+    private RedactionLog redactionLog;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 
 import lombok.Data;
@@ -23,4 +24,6 @@ public class Document {
     private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
     private boolean headlines;
+
+    private List<RedactionLogEntity> redactionLogEntities = new ArrayList<>();
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Paragraph.java

@@ -16,6 +16,7 @@ import lombok.NoArgsConstructor;
 public class Paragraph {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
+    private String headLine;
 
     public SearchableText getSearchableText(){
         SearchableText searchableText = new SearchableText();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -8,6 +8,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
@@ -48,10 +49,10 @@ public class RedactionController implements RedactionResource {
 
             if (redactionRequest.isFlatRedaction()) {
                 PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
-                return convert(flatDocument, classifiedDoc.getPages().size());
+                return convert(flatDocument, classifiedDoc.getPages().size(), new RedactionLog(classifiedDoc.getRedactionLogEntities()));
             }
 
-            return convert(pdDocument, classifiedDoc.getPages().size());
+            return convert(pdDocument, classifiedDoc.getPages().size(), new RedactionLog(classifiedDoc.getRedactionLogEntities()));
 
         } catch (IOException e) {
             throw new RedactionException(e);
@@ -68,7 +69,7 @@ public class RedactionController implements RedactionResource {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
 
-            return convert(pdDocument, classifiedDoc.getPages().size());
+            return convert(pdDocument, classifiedDoc.getPages().size(), null);
 
         } catch (IOException e) {
             throw new RedactionException(e);
@@ -85,7 +86,7 @@ public class RedactionController implements RedactionResource {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
 
-            return convert(pdDocument, classifiedDoc.getPages().size());
+            return convert(pdDocument, classifiedDoc.getPages().size(), null);
 
         } catch (IOException e) {
             throw new RedactionException(e);
@@ -124,16 +125,17 @@ public class RedactionController implements RedactionResource {
         droolsExecutionService.updateRules(rules);
     }
 
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+    private RedactionResult convert(PDDocument document, int numberOfPages, RedactionLog redactionLog) throws IOException {
 
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
             return RedactionResult.builder()
                     .document(byteArrayOutputStream.toByteArray())
                     .numberOfPages(numberOfPages)
+                    .redactionLog(redactionLog)
                     .build();
         }
 
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -16,19 +16,24 @@ public class Entity {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private Integer start;
     private Integer end;
+    private String headline;
+    private int matchedRule;
 
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences) {
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule) {
         this.word = word;
         this.type = type;
         this.redaction = redaction;
         this.redactionReason = redactionReason;
         this.positionSequences = positionSequences;
+        this.headline = headline;
+        this.matchedRule = matchedRule;
     }
 
-    public Entity(String word, String type, Integer start, Integer end) {
+    public Entity(String word, String type, Integer start, Integer end, String headline) {
         this.word = word;
         this.type = type;
         this.start = start;
         this.end = end;
+        this.headline = headline;
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -23,6 +23,8 @@ public class Section {
     //This does not contain linebreaks and must always be used for correct offsets.
     private String searchText;
 
+    private String headline;
+
     public boolean contains(String type) {
         return entities.stream().anyMatch(entity -> entity.getType().equals(type));
     }
@@ -31,7 +33,8 @@ public class Section {
         entities.forEach(entity -> {
             if(entity.getType().equals(type)){
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
@@ -40,7 +43,8 @@ public class Section {
         entities.forEach(entity -> {
             if(entity.getType().equals(type)){
                 entity.setRedaction(false);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
@@ -67,7 +71,8 @@ public class Section {
         entities.forEach(entity -> {
             if(entity.getType().equals(asType)){
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
 
@@ -88,7 +93,8 @@ public class Section {
         entities.forEach(entity -> {
             if(entity.getType().equals(asType)){
                 entity.setRedaction(true);
-                entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
             }
         });
     }
@@ -109,7 +115,7 @@ public class Section {
                     if (startIndex > -1 &&
                             (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText.charAt(startIndex - 1))) &&
                             (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
-                        found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex));
+                        found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
                     }
                 } while (startIndex > -1);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -27,6 +27,7 @@ public class EntityRedactionService {
     private final DictionaryService dictionaryService;
     private final DroolsExecutionService droolsExecutionService;
 
+
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
@@ -55,12 +56,13 @@ public class EntityRedactionService {
                 }
             }
 
-            Set<Entity> entities = findEntities(searchableText);
+            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadLine());
             Section analysedSection = droolsExecutionService.executeRules(Section
                     .builder()
                     .entities(entities)
                     .text(searchableText.getAsStringWithLinebreaks())
                     .searchText(searchableText.toString())
+                    .headline(paragraph.getHeadLine())
                     .build());
 
             for (Entity entity : analysedSection.getEntities()) {
@@ -70,13 +72,14 @@ public class EntityRedactionService {
             documentEntities.addAll(analysedSection.getEntities());
 
             for (SearchableText searchableRow : searchableRows) {
-                Set<Entity> rowEntities = findEntities(searchableRow);
+                Set<Entity> rowEntities = findEntities(searchableRow, "//TODO TableHeader");
 
                 Section analysedRowSection = droolsExecutionService.executeRules(Section
                         .builder()
                         .entities(rowEntities)
                         .text(searchableRow.getAsStringWithLinebreaks())
                         .searchText(searchableRow.toString())
+                        .headline("//TODO TableHeader")
                         .build());
 
                 for (Entity entity : analysedRowSection.getEntities()) {
@@ -89,13 +92,14 @@ public class EntityRedactionService {
         documentEntities.forEach(entity -> {
             entity.getPositionSequences().forEach(sequence -> {
                 classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
-                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence))
+                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence), entity.getHeadline(), entity.getMatchedRule())
                 );
             });
         });
     }
 
-    private Set<Entity> findEntities(SearchableText searchableText) {
+
+    private Set<Entity> findEntities(SearchableText searchableText, String headline) {
 
         String normalizedInputString = searchableText.toString();
 
@@ -111,7 +115,7 @@ public class EntityRedactionService {
                     if (startIndex > -1 &&
                             (startIndex == 0 || Character.isWhitespace(normalizedInputString.charAt(startIndex - 1)) || isSeparator(normalizedInputString.charAt(startIndex - 1))) &&
                             (stopIndex == normalizedInputString.length() || isSeparator(normalizedInputString.charAt(stopIndex)))) {
-                        found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex));
+                        found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex, headline));
                     }
                 } while (startIndex > -1);
             }
@@ -126,6 +130,7 @@ public class EntityRedactionService {
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
+
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
@@ -138,4 +143,5 @@ public class EntityRedactionService {
         entities.removeAll(wordsToRemove);
     }
 
-}
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -25,6 +25,7 @@ public class SectionsBuilderService {
 
         AbstractTextContainer prev = null;
 
+        String lastHeadline = "";
         for (Page page : document.getPages()) {
             for (AbstractTextContainer current : page.getTextBlocks()) {
 
@@ -37,6 +38,8 @@ public class SectionsBuilderService {
                 if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
 
                     Paragraph cb1 = buildTextBlock(chunkWords);
+                    cb1.setHeadLine(lastHeadline);
+                    lastHeadline = current.getText();
                     chunkBlockList1.add(cb1);
                     chunkWords = new ArrayList<>();
 
@@ -51,6 +54,7 @@ public class SectionsBuilderService {
         Paragraph cb1 = buildTextBlock(chunkWords);
         if (cb1 != null) {
             chunkBlockList1.add(cb1);
+            cb1.setHeadLine(lastHeadline);
         }
 
         document.setParagraphs(chunkBlockList1);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -20,6 +20,9 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Position;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntity;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -77,6 +80,8 @@ public class AnnotationHighlightService {
 
             for (Entity entity : classifiedDoc.getEntities().get(page)) {
 
+                RedactionLogEntity redactionLogEntity = new RedactionLogEntity();
+
                 for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
 
                     if (flatRedaction && !isRedactionType(entity)) {
@@ -91,7 +96,7 @@ public class AnnotationHighlightService {
                         float posXEnd;
                         float posYInit;
                         float posYEnd;
-                        float[] quadPoints;
+                        Position textHighlightPosition;
 
                         if (textPositions.getTextPositions().get(0).getRotation() == 90) {
 
@@ -100,14 +105,16 @@ public class AnnotationHighlightService {
                             posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
                             posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 2;
 
-                            quadPoints = new float[]{posXInit, posYInit, posXInit, posYEnd + height + 2, posXEnd, posYInit, posXEnd, posYEnd + height + 2};
+                            textHighlightPosition = new Position(new Point(posXInit, posYInit), new Point(posXInit, posYEnd + height + 2), new Point(posXEnd, posYInit), new Point(posXEnd, posYEnd + height + 2), page);
+
                         } else {
 
                             posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
                             posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
                             posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj();
                             posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj();
-                            quadPoints = new float[]{posXInit, posYEnd + height + 2, posXEnd, posYEnd + height + 2, posXInit, posYInit - 2, posXEnd, posYEnd - 2};
+
+                            textHighlightPosition = new Position(new Point(posXInit, posYEnd + height + 2), new Point(posXEnd, posYEnd + height + 2), new Point(posXInit, posYInit - 2), new Point(posXEnd, posYEnd - 2), page);
                         }
 
 
@@ -115,23 +122,21 @@ public class AnnotationHighlightService {
                         PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
                         highlight.constructAppearances();
 
-                        PDRectangle position = new PDRectangle();
+                        PDRectangle annotationPosition = new PDRectangle();
-                        position.setLowerLeftX(posXInit);
+                        annotationPosition.setLowerLeftX(posXInit);
-                        position.setLowerLeftY(posYEnd);
+                        annotationPosition.setLowerLeftY(posYEnd);
-                        position.setUpperRightX(posXEnd);
+                        annotationPosition.setUpperRightX(posXEnd);
-                        position.setUpperRightY(posYEnd + height);
+                        annotationPosition.setUpperRightY(posYEnd + height);
 
-                        highlight.setRectangle(position);
+                        highlight.setRectangle(annotationPosition);
                         if (!flatRedaction) {
                             highlight.setAnnotationName(entityPositionSequence.getId().toString());
                             highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents(entity.getRedactionReason());
+                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" +entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity.getHeadline() + "\"");
                         }
 
-                        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
-                        // of the area to be highlighted
 
-                        highlight.setQuadPoints(quadPoints);
+                        highlight.setQuadPoints(new float[]{textHighlightPosition.getTopLeft().getX(), textHighlightPosition.getTopLeft().getY(), textHighlightPosition.getTopRight().getX(), textHighlightPosition.getTopRight().getY(), textHighlightPosition.getBottomLeft().getX(), textHighlightPosition.getBottomLeft().getY(), textHighlightPosition.getBottomRight().getX(), textHighlightPosition.getBottomRight().getY()});
 
                         PDColor color;
                         if (flatRedaction) {
@@ -142,8 +147,19 @@ public class AnnotationHighlightService {
 
                         highlight.setColor(color);
                         annotations.add(highlight);
+
+                        redactionLogEntity.getPositions().add(textHighlightPosition);
+
                     }
+                    redactionLogEntity.setId(entityPositionSequence.getId().toString());
                 }
+                redactionLogEntity.setColor(getColor(entity));
+                redactionLogEntity.setReason(entity.getRedactionReason());
+                redactionLogEntity.setValue(entity.getWord());
+                redactionLogEntity.setType(entity.getType());
+                redactionLogEntity.setRedacted(entity.isRedaction());
+                redactionLogEntity.setSection(entity.getHeadline());
+                classifiedDoc.getRedactionLogEntities().add(redactionLogEntity);
             }
 
         }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -174,4 +174,4 @@ public class RedactionIntegrationTest {
 
     }
 
-}
+}