Fixed table extraction problems

Merge in RED/redaction-service from ExpandAuthors to master

* commit '35f3582d0850053c1dc38ef0b149daa47bd9cc56':
  Expand CBI Authors with firstname initials

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java

@@ -34,7 +34,6 @@ import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.cos.COSNumber;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
 import org.apache.pdfbox.text.PDFTextStripper;
@@ -55,10 +54,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     protected PDPage pdpage;
 
     @Getter
-    private int maxCharWidths;
-
-    @Getter
-    private int maxCharHeight;
+    private int minCharWidths;
 
     @Getter
     private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@@ -216,7 +212,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
                         .getWidth(), (float) imageBounds.getHeight());
 
-                this.imageBounds.add(rect);
+                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
+                    this.imageBounds.add(rect);
+                }
             }
         } catch (Exception e) {
             log.warn("Problem during image extraction: {}", e.getMessage());
@@ -234,36 +232,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
         imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
         imageTransform.translate(0, -pdfImage.getHeight());
 
-        AffineTransform pageTransform = createCurrentPageTransformation();
+        AffineTransform pageTransform = new AffineTransform();
         pageTransform.concatenate(imageTransform);
 
         return pageTransform.createTransformedShape(imageBounds).getBounds2D();
     }
 
 
-    protected AffineTransform createCurrentPageTransformation() {
-
-        PDRectangle cb = pdpage.getCropBox();
-        AffineTransform pageTransform = new AffineTransform();
-
-        switch (pdpage.getRotation()) {
-            case 90:
-                pageTransform.translate(cb.getHeight(), 0);
-                break;
-            case 180:
-                pageTransform.translate(cb.getWidth(), cb.getHeight());
-                break;
-            case 270:
-                pageTransform.translate(0, cb.getWidth());
-                break;
-        }
-
-        pageTransform.rotate(Math.toRadians(pdpage.getRotation()));
-
-        return pageTransform;
-    }
-
-
     private float floatValue(COSBase value) {
 
         if (value instanceof COSNumber) {
@@ -304,14 +279,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
         int startIndex = 0;
         for (int i = 0; i <= textPositions.size() - 1; i++) {
 
-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if (charHeight > maxCharHeight) {
-                maxCharHeight = charHeight;
-            }
-
             int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if (charWidth > maxCharWidths) {
-                maxCharWidths = charWidth;
+            if (charWidth < minCharWidths) {
+                minCharWidths = charWidth;
             }
 
             if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
@@ -363,8 +333,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     @Override
     public String getText(PDDocument doc) throws IOException {
 
-        maxCharWidths = 0;
-        maxCharWidths = 0;
+        minCharWidths = Integer.MAX_VALUE;
         textPositionSequences.clear();
         imageBounds = new ArrayList<>();
         rulings.clear();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java

@@ -19,6 +19,5 @@ public class ParsedElements {
     private boolean landscape;
     private boolean rotated;
 
-    private float maxCharWidth;
-    private float maxCharHeight;
+    private float minCharWidth;
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -62,9 +62,10 @@ public class Section {
     }
 
 
-    public boolean hasTableHeader(String headerName){
-       String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
-       return tabularData != null && tabularData.containsKey(cleanHeaderName);
+    public boolean hasTableHeader(String headerName) {
+
+        String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+        return tabularData != null && tabularData.containsKey(cleanHeaderName);
     }
 
 
@@ -80,6 +81,34 @@ public class Section {
     }
 
 
+    public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
+                            .getRedactionReason(), entity.getLegalBasis()));
+
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
     public void redact(String type, int ruleNumber, String reason, String legalBasis) {
 
         boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -160,7 +189,7 @@ public class Section {
 
                 String trimmedValue = value.trim();
                 String cleanValue;
-                if(trimmedValue.startsWith(":")){
+                if (trimmedValue.startsWith(":")) {
                     cleanValue = trimmedValue.substring(1).trim();
                 } else {
                     cleanValue = trimmedValue;
@@ -207,8 +236,8 @@ public class Section {
     }
 
 
-    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
-                              String reason, String legalBasis) {
+    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
+                                          int ruleNumber, String reason, String legalBasis) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -363,6 +392,7 @@ public class Section {
             }
         }
     }
+
 }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -61,14 +61,13 @@ public class PdfSegmentationService {
                     .rulings(stripper.getRulings())
                     .sequences(stripper.getTextPositionSequences())
                     .imageBounds(stripper.getImageBounds())
-                    .maxCharWidth(stripper.getMaxCharWidths())
-                    .maxCharHeight(stripper.getMaxCharWidths())
+                    .minCharWidth(stripper.getMinCharWidths())
                     .landscape(isLandscape)
                     .rotated(isRotated)
                     .build();
 
             CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
-                    .getMaxCharWidth(), parsedElements.getMaxCharHeight());
+                    .getMinCharWidth());
 
             Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
                     .getVertical());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java

@@ -18,9 +18,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 @Service
 public class RulingCleaningService {
 
-    public CleanRulings getCleanRulings(List<Ruling> rulings, float maxCharWidth, float maxCharHeight){
+    public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth){
         if (!rulings.isEmpty()) {
-            snapPoints(rulings, maxCharWidth , maxCharHeight);
+            snapPoints(rulings, minCharWidth , minCharWidth);
         }
 
         List<Ruling> vrs = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -441,7 +441,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("redactionTest");
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_08_Volume_3CA_B-6_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
 
         AnalyzeRequest request = AnalyzeRequest.builder()
                 .ruleSetId(TEST_RULESET_ID)

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java

@@ -1,15 +1,36 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+
 public class RegExPatternTest {
 
 
+    @Test
+    public void testExpand(){
+        String pattern = "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)";
+        String text = ", G.R., Simoneaux,";
+        Pattern compiledPattern = Pattern.compile(pattern, 0);
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(1);
+            if (StringUtils.isNotBlank(match)) {
+                System.out.println(match);
+            }
+        }
+    }
+
+
+
     @Test
     public void testEmailRegEx(){
         String text = "Address: Schwarzwaldalle " +

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl

@@ -7,6 +7,15 @@ global Section section
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redact CBI Authors"
     when
         Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -7,6 +7,15 @@ global Section section
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redacted because Section contains Vertebrate"
     when
         Section(matchesType("vertebrate"))