Fixed table extraction problems

Pull request #115 : Expand CBI Authors with firstname initials
Merge in RED/redaction-service from ExpandAuthors to master * commit '35f3582d0850053c1dc38ef0b149daa47bd9cc56': Expand CBI Authors with firstname initials
2021-02-03 14:34:29 +01:00 · 2021-02-02 12:35:07 +01:00 · 2021-02-02 12:20:40 +01:00 · 2021-02-01 16:17:19 +01:00 · 2021-02-01 16:04:45 +01:00 · 2021-02-01 15:46:57 +01:00
15 changed files with 272 additions and 56 deletions
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -16,6 +17,8 @@ public class Page {
    @NonNull
    private List<AbstractTextContainer> textBlocks;

+    private List<Rectangle2D> imageBounds;
+
    private Rectangle bodyTextFrame;

    private boolean landscape;
@@ -64,8 +64,12 @@ public class RedactionController implements RedactionResource {

            log.info("Redaction analysis successful...");

-            return AnalyzeResult.builder().sectionGrid(classifiedDoc.getSectionGrid()).redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
-                    .getRulesVersion(), analyzeRequest.getRuleSetId())).build();
+            return AnalyzeResult.builder()
+                    .sectionGrid(classifiedDoc.getSectionGrid())
+                    .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
+                    .getRulesVersion(), analyzeRequest.getRuleSetId()))
+                    .numberOfPages(classifiedDoc.getPages().size())
+                    .build();

        } catch (Exception e) {
            throw new RedactionException(e);
@@ -1,6 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.parsing;

+import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -28,10 +30,15 @@ import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
 import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.cos.COSNumber;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.graphics.PDXObject;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
 import org.apache.pdfbox.text.PDFTextStripper;
 import org.apache.pdfbox.text.TextPosition;
+import org.apache.pdfbox.util.Matrix;

 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@@ -43,11 +50,11 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {

-    @Getter
-    private int maxCharWidths;
+    @Setter
+    protected PDPage pdpage;

    @Getter
-    private int maxCharHeight;
+    private int minCharWidths;

    @Getter
    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@@ -57,13 +64,18 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    private final List<Ruling> graphicsPath = new ArrayList<>();

+    @Getter
+    private List<Rectangle2D> imageBounds = new ArrayList<>();
+
    private float path_x;
    private float path_y;

    @Setter
    private int pageNumber;

+
    public PDFLinesTextStripper() throws IOException {
+
        super();
        this.addOperator(new SetStrokingColorSpace());
        this.addOperator(new SetNonStrokingColorSpace());
@@ -87,9 +99,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetLineWidth());
    }

+
    @Override
-    protected void processOperator(Operator operator, List<COSBase> arguments)
-            throws IOException {
+    protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {

        String operation = operator.getName();

@@ -110,9 +122,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {

                    // The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
                    if (pos.getY() > path_y) {
-                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
+                                .getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
+                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
+                                .getX(), path_y)));
                    }

                    path_x = (float) pos.getX();
@@ -133,19 +147,25 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    Point2D p2 = transformPosition(x + width, y + height);

                    // Horizontal lines
-                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
-                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
+                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
+                            .getX(), (float) p1.getY())));
+                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
+                            .getX(), (float) p2.getY())));

                    // Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
                    if (p2.getY() > p1.getY()) {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
+                                .getX(), (float) p2.getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
+                                .getX(), (float) p1.getY())));
                    }
                    if (p2.getY() > p1.getY()) {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
+                                .getX(), (float) p2.getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
+                                .getX(), (float) p1.getY())));
                    }
                }
                break;
@@ -168,12 +188,59 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            case OperatorName.ENDPATH:
                graphicsPath.clear();
                break;
+
+            case OperatorName.DRAW_OBJECT:
+                processImageOperation(arguments);
+                break;
+
        }

        super.processOperator(operator, arguments);
    }

+
+    protected void processImageOperation(List<COSBase> arguments) {
+
+        try {
+            COSName objectName = (COSName) arguments.get(0);
+            PDXObject xobject = getResources().getXObject(objectName);
+            if (xobject instanceof PDImageXObject) {
+                PDImageXObject pdfImage = (PDImageXObject) xobject;
+
+                Rectangle2D imageBounds = calculateImagePosition(pdfImage);
+
+                Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
+                        .getWidth(), (float) imageBounds.getHeight());
+
+                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
+                    this.imageBounds.add(rect);
+                }
+            }
+        } catch (Exception e) {
+            log.warn("Problem during image extraction: {}", e.getMessage());
+        }
+    }
+
+
+    private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
+
+        Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
+
+        Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
+
+        AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
+        imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
+        imageTransform.translate(0, -pdfImage.getHeight());
+
+        AffineTransform pageTransform = new AffineTransform();
+        pageTransform.concatenate(imageTransform);
+
+        return pageTransform.createTransformedShape(imageBounds).getBounds2D();
+    }
+
+
    private float floatValue(COSBase value) {
+
        if (value instanceof COSNumber) {
            return ((COSNumber) value).floatValue();
        } else {
@@ -181,38 +248,45 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        }
    }

+
    private Point2D.Float transformPosition(float x, float y) {
+
        return super.transformedPoint(x, y);
    }

+
    private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {

        try {
-            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
+            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
+                    .toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor()
+                    .isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
                rulings.addAll(path);
            }
        } catch (UnsupportedOperationException e) {
-            log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
+            log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
+                    .getColorSpace()
+                    .getName() + " or " + getGraphicsState().getNonStrokingColor()
+                    .getColorSpace()
+                    .getName() + " does not support toRGB");
        }
    }

+
    @Override
    public void writeString(String text, List<TextPosition> textPositions) throws IOException {

        int startIndex = 0;
        for (int i = 0; i <= textPositions.size() - 1; i++) {

-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if(charHeight > maxCharHeight){
-                maxCharHeight = charHeight;
-            }
-
            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if(charWidth > maxCharWidths){
-                maxCharWidths = charWidth;
+            if (charWidth < minCharWidths) {
+                minCharWidths = charWidth;
            }

-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
+            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0"))) {
                startIndex++;
                continue;
            }
@@ -220,15 +294,21 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i;
            }

-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
+            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i + 1;
@@ -236,21 +316,26 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        }

        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
+                .getUnicode()
+                .equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
            sublist = sublist.subList(0, sublist.size() - 1);
        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
+                .getUnicode()
+                .equals("\u00A0")))) {
            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
        }
        super.writeString(text);
    }

+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        maxCharWidths = 0;
-        maxCharWidths = 0;
+        minCharWidths = Integer.MAX_VALUE;
        textPositionSequences.clear();
+        imageBounds = new ArrayList<>();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;

+import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@@ -13,10 +14,10 @@ public class ParsedElements {

    private List<TextPositionSequence> sequences;
    private List<Ruling> rulings;
+    private List<Rectangle2D> imageBounds;

    private boolean landscape;
    private boolean rotated;

-    private float maxCharWidth;
-    private float maxCharHeight;
+    private float minCharWidth;
 }
@@ -62,9 +62,10 @@ public class Section {
    }


-    public boolean hasTableHeader(String headerName){
-       String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
-       return tabularData != null && tabularData.containsKey(cleanHeaderName);
+    public boolean hasTableHeader(String headerName) {
+
+        String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+        return tabularData != null && tabularData.containsKey(cleanHeaderName);
    }


@@ -80,6 +81,34 @@ public class Section {
    }


+    public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
+                            .getRedactionReason(), entity.getLegalBasis()));
+
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
    public void redact(String type, int ruleNumber, String reason, String legalBasis) {

        boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -160,7 +189,7 @@ public class Section {

                String trimmedValue = value.trim();
                String cleanValue;
-                if(trimmedValue.startsWith(":")){
+                if (trimmedValue.startsWith(":")) {
                    cleanValue = trimmedValue.substring(1).trim();
                } else {
                    cleanValue = trimmedValue;
@@ -207,8 +236,8 @@ public class Section {
    }


-    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
-                              String reason, String legalBasis) {
+    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
+                                          int ruleNumber, String reason, String legalBasis) {

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);

@@ -363,6 +392,7 @@ public class Section {
            }
        }
    }
+
 }


@@ -53,25 +53,24 @@ public class AnnotationService {

            List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
            if (logEntries != null && !logEntries.isEmpty()) {
-                addAnnotations(logEntries, pdPage, page, redactionLog.getRuleSetId());
+                addAnnotations(logEntries, pdPage, page);
            }
        }
    }


-    private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page,
-                                String ruleSetId) throws IOException {
+    private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {

        List<PDAnnotation> annotations = pdPage.getAnnotations();

        for (RedactionLogEntry entry : logEntries) {
-            annotations.addAll(createAnnotation(entry, page, ruleSetId, pdPage.getMediaBox(), pdPage.getCropBox()));
+            annotations.addAll(createAnnotation(entry, page, pdPage.getMediaBox(), pdPage.getCropBox()));
        }
    }


-    private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, String ruleSetId,
-                                                PDRectangle mediaBox, PDRectangle cropBox) {
+    private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, PDRectangle mediaBox,
+                                                PDRectangle cropBox) {

        List<PDAnnotation> annotations = new ArrayList<>();

@@ -89,7 +88,7 @@ public class AnnotationService {
        PDRectangle pdRectangle = toPDRectangle(rectangles, mediaBox, cropBox);
        annotation.setRectangle(pdRectangle);
        annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
-        if (!dictionaryService.isHint(redactionLogEntry.getType(), ruleSetId)) {
+        if (!redactionLogEntry.isHint()) {
            annotation.setContents(createAnnotationContent(redactionLogEntry));
        }
        annotation.setTitlePopup(redactionLogEntry.getId());
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -27,6 +28,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -37,6 +39,8 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class RedactionLogCreatorService {

+    private static final String IMAGE = "image";
+
    private final DictionaryService dictionaryService;


@@ -56,6 +60,30 @@ public class RedactionLogCreatorService {
            if (manualRedactionPages.contains(page)) {
                addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
            }
+
+            if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
+                addImageEntries(classifiedDoc, page, ruleSetId);
+            }
+        }
+    }
+
+
+    private void addImageEntries(Document classifiedDoc, int pageNumber, String ruleSetId) {
+
+        for (Rectangle2D imageBounds : classifiedDoc.getPages().get(pageNumber - 1).getImageBounds()) {
+            RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
+                    .id(IdBuilder.buildId(imageBounds, pageNumber))
+                    .color(getColor(IMAGE, ruleSetId))
+                    .type(IMAGE)
+                    .redacted(false)
+                    .isHint(true)
+                    .manual(false)
+                    .isDictionaryEntry(false)
+                    .isRecommendation(false)
+                    .positions(List.of(new Rectangle(new Point((float) imageBounds.getX(), (float) imageBounds.getY()), (float) imageBounds
+                            .getWidth(), (float) imageBounds.getHeight(), pageNumber)))
+                    .build();
+            classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
        }
    }

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import java.awt.geom.Rectangle2D;
 import java.nio.charset.StandardCharsets;
 import java.util.List;

@@ -23,4 +24,16 @@ public class IdBuilder {

        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
    }
+
+
+    public String buildId(Rectangle2D rectangle2D, int page){
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("x").append(rectangle2D.getX()).append("y").append(rectangle2D.getY()).append("h").append(rectangle2D.getHeight()).append("w").append(rectangle2D.getWidth()).append("p").append(page);
+
+        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
+    }
+
+
+
 }
@@ -48,6 +48,7 @@ public class PdfSegmentationService {
            stripper.setPageNumber(pageNumber);
            stripper.setStartPage(pageNumber);
            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
            stripper.getText(pdDocument);

            PDRectangle pdr = pdPage.getMediaBox();
@@ -56,18 +57,17 @@ public class PdfSegmentationService {
            int rotation = pdPage.getRotation();
            boolean isRotated = rotation != 0 && rotation != 360;

-
            ParsedElements parsedElements = ParsedElements.builder()
                    .rulings(stripper.getRulings())
                    .sequences(stripper.getTextPositionSequences())
-                    .maxCharWidth(stripper.getMaxCharWidths())
-                    .maxCharHeight(stripper.getMaxCharWidths())
+                    .imageBounds(stripper.getImageBounds())
+                    .minCharWidth(stripper.getMinCharWidths())
                    .landscape(isLandscape)
                    .rotated(isRotated)
                    .build();

            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
-                    .getMaxCharWidth(), parsedElements.getMaxCharHeight());
+                    .getMinCharWidth());

            Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
                    .getVertical());
@@ -81,8 +81,10 @@ public class PdfSegmentationService {

            page.setPageNumber(pageNumber);
            increaseDocumentStatistics(page, document);
+            page.setImageBounds(parsedElements.getImageBounds());
            pages.add(page);
        }
+
        document.setPages(pages);

        classificationService.classifyDocument(document);
@@ -90,11 +92,9 @@ public class PdfSegmentationService {
        sectionsBuilderService.buildSections(document);

        return document;
-
    }


-
    private void increaseDocumentStatistics(Page page, Document document) {

        if (!page.isLandscape()) {
@@ -18,9 +18,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
@Service
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<Ruling> rulings, float maxCharWidth, float maxCharHeight){
+    public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth){
        if (!rulings.isEmpty()) {
-            snapPoints(rulings, maxCharWidth , maxCharHeight);
+            snapPoints(rulings, minCharWidth , minCharWidth);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -81,6 +81,7 @@ public class RedactionIntegrationTest {
    private static final String PUBLISHED_INFORMATION = "published_information";
    private static final String TEST_METHOD = "test_method";
    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";

    private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
    private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -157,6 +158,7 @@ public class RedactionIntegrationTest {
        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
+        when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
        when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
    }

@@ -238,6 +240,11 @@ public class RedactionIntegrationTest {
                        .stream()
                        .map(this::cleanDictionaryEntry)
                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/image.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
    }


@@ -264,6 +271,7 @@ public class RedactionIntegrationTest {
        typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
        typeColorMap.put(FALSE_POSITIVE, "#ffffff");
        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");

        hintTypeMap.put(VERTEBRATE, true);
        hintTypeMap.put(ADDRESS, false);
@@ -280,6 +288,7 @@ public class RedactionIntegrationTest {
        hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
        hintTypeMap.put(FALSE_POSITIVE, true);
        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);

        caseInSensitiveMap.put(VERTEBRATE, true);
        caseInSensitiveMap.put(ADDRESS, false);
@@ -296,6 +305,7 @@ public class RedactionIntegrationTest {
        caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
        caseInSensitiveMap.put(FALSE_POSITIVE, false);
        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);

        recommendationTypeMap.put(VERTEBRATE, false);
        recommendationTypeMap.put(ADDRESS, false);
@@ -312,6 +322,8 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
        recommendationTypeMap.put(FALSE_POSITIVE, false);
        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+

        rankTypeMap.put(FALSE_POSITIVE, 160);
        rankTypeMap.put(PURITY, 155);
@@ -328,6 +340,8 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(HINT_ONLY, 50);
        rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
+        rankTypeMap.put(IMAGE, 30);
+

        colors.setDefaultColor("#acfc00");
        colors.setNotRedacted("#cccccc");
@@ -427,7 +441,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");

        AnalyzeRequest request = AnalyzeRequest.builder()
                .ruleSetId(TEST_RULESET_ID)
@@ -1,15 +1,36 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;

 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+
 public class RegExPatternTest {


+    @Test
+    public void testExpand(){
+        String pattern = "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)";
+        String text = ", G.R., Simoneaux,";
+        Pattern compiledPattern = Pattern.compile(pattern, 0);
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(1);
+            if (StringUtils.isNotBlank(match)) {
+                System.out.println(match);
+            }
+        }
+    }
+
+
+
    @Test
    public void testEmailRegEx(){
        String text = "Address: Schwarzwaldalle " +
@@ -7,6 +7,15 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redact CBI Authors"
    when
        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
@@ -7,6 +7,15 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redacted because Section contains Vertebrate"
    when
        Section(matchesType("vertebrate"))
Author	SHA1	Message	Date
Dominique Eifländer	fc2ac03691	Fixed table extraction problems	2021-02-03 14:34:29 +01:00
Dominique Eiflaender	7898f6a30f	Pull request #115 : Expand CBI Authors with firstname initials Merge in RED/redaction-service from ExpandAuthors to master * commit '35f3582d0850053c1dc38ef0b149daa47bd9cc56': Expand CBI Authors with firstname initials	2021-02-02 12:35:07 +01:00
Dominique Eifländer	35f3582d08	Expand CBI Authors with firstname initials	2021-02-02 12:20:40 +01:00
Dominique Eiflaender	b4699caaae	Pull request #114 : Fixed position problem for rotated images Merge in RED/redaction-service from rotatedImages to master * commit 'd8e444280b5a025e19785c70c9fcbb11480b5042': Fixed position problem for rotated images	2021-02-01 16:17:19 +01:00
Dominique Eifländer	d8e444280b	Fixed position problem for rotated images	2021-02-01 16:04:45 +01:00
Dominique Eiflaender	ec4db538a7	Pull request #113 : Ignore too small images Merge in RED/redaction-service from IgnoreTooSmall to master * commit '39ca191b9cb5dec5fbb01e8319cbbe361c7cd85e': Ignore too small images	2021-02-01 15:46:57 +01:00
Dominique Eifländer	39ca191b9c	Ignore too small images	2021-02-01 15:14:46 +01:00
Dominique Eiflaender	0cec88f1b4	Pull request #112 : Annotate images Merge in RED/redaction-service from annotateImages to master * commit 'acddfafa5b60d0120a48bdf47fe218bf59e359d2': Annotate images	2021-02-01 14:12:28 +01:00
Dominique Eifländer	acddfafa5b	Annotate images	2021-02-01 13:52:18 +01:00
Dominique Eiflaender	6b68890fcf	Pull request #111 : RED-1019: Fixed returning numberOfPages Merge in RED/redaction-service from RED-1019 to master * commit '1ed99412595b560d92e542d949e52d19d58beb2c': RED-1019: Fixed returning numberOfPages	2021-01-29 13:32:05 +01:00
Dominique Eifländer	1ed9941259	RED-1019: Fixed returning numberOfPages	2021-01-29 13:17:25 +01:00
Timo Bejan	1e7d535a33	Pull request #110 : fixed issue affecting service on restarts Merge in RED/redaction-service from bugfix-annotate to master * commit '1091d6a886243387f23da32d0c678b66b0c21f26': fixed issue affecting service on restarts	2021-01-28 10:17:02 +01:00