From ad265f9256394a187115f0166951573c415ef202 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Wed, 19 Oct 2022 17:16:23 +0200 Subject: [PATCH] RED-5381: Use new codestyle --- .../v1/server/classification/model/Page.java | 15 +- .../classification/model/TextBlock.java | 101 +++-- .../service/BlockificationService.java | 93 ++-- .../service/ClassificationService.java | 95 ++--- .../utils/CoordSystemHelper.java | 9 +- .../classification/utils/PositionUtils.java | 15 +- .../parsing/model/TextPositionSequence.java | 26 +- .../segmentation/PdfSegmentationService.java | 12 +- .../model/AbstractTextContainer.java | 12 +- .../service/TableExtractionService.java | 43 +- .../service/PdfVisualisationService.java | 30 +- .../v1/server/RedactionIntegrationTest.java | 398 +++++------------- 12 files changed, 338 insertions(+), 511 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java index f34c0799..2cc88f86 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java @@ -1,17 +1,18 @@ package com.iqser.red.service.redaction.v1.server.classification.model; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; -import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; -import lombok.Data; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; - import java.util.ArrayList; import java.util.List; import org.apache.pdfbox.pdmodel.common.PDRectangle; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; + +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + @Data @RequiredArgsConstructor public class Page { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java index ab1c075e..e74f5ec3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java @@ -1,5 +1,8 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import java.util.ArrayList; +import java.util.List; + import com.dslplatform.json.CompiledJson; import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -7,14 +10,12 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.ArrayList; -import java.util.List; - @AllArgsConstructor @Builder @Data @@ -46,20 +47,24 @@ public class TextBlock extends AbstractTextContainer { @JsonIgnore private float highestFontSize; - @JsonIgnore private String classification; - private TextDirection getDir(){ + private TextDirection getDir() { + return sequences.get(0).getDir(); } - private float getPageHeight(){ + + private float getPageHeight() { + return sequences.get(0).getPageHeight(); } - private float getPageWidth(){ + + private float getPageWidth() { + return sequences.get(0).getPageWidth(); } @@ -70,16 +75,13 @@ public class TextBlock extends AbstractTextContainer { if (getDir().getDegrees() == 90) { return minY; - } - else if (getDir().getDegrees() == 180) { - return getPageWidth() - maxX; + } else if (getDir().getDegrees() == 180) { + return getPageWidth() - maxX; - } - else if (getDir().getDegrees() == 270) { + } else if (getDir().getDegrees() == 270) { - return getPageWidth() - maxY ; - } - else { + return getPageWidth() - maxY; + } else { return minX; } } @@ -91,44 +93,30 @@ public class TextBlock extends AbstractTextContainer { if (getDir().getDegrees() == 90) { return maxY; - } + } else if (getDir().getDegrees() == 180) { + return getPageWidth() - minX; + } else if (getDir().getDegrees() == 270) { + return getPageWidth() - minY; - else if (getDir().getDegrees() == 180) { - return getPageWidth() - minX; - } - - else if (getDir().getDegrees() == 270) { - return getPageWidth() - minY ; - - } - - else { + } else { return maxX; } } - - - @JsonIgnore @JsonAttribute(ignore = true) public float getY1() { - if (getDir().getDegrees() == 90 ) { - return minX; - } - - else if (getDir().getDegrees() == 180) { + if (getDir().getDegrees() == 90) { + return minX; + } else if (getDir().getDegrees() == 180) { return maxY; - } - - else if (getDir().getDegrees() == 270) { + } else if (getDir().getDegrees() == 270) { return getPageHeight() - maxX; - } - else { + } else { return getPageHeight() - maxY; } } @@ -139,28 +127,20 @@ public class TextBlock extends AbstractTextContainer { public float getY2() { if (getDir().getDegrees() == 90) { - return maxX; - } - - else if (getDir().getDegrees() == 180 ) { + return maxX; + } else if (getDir().getDegrees() == 180) { return minY; - } - - else if (getDir().getDegrees() == 270 ) { - return getPageHeight() - minX; - } - - else { + } else if (getDir().getDegrees() == 270) { + return getPageHeight() - minX; + } else { return getPageHeight() - minY; } } - - - public TextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation) { + this.minX = minX; this.maxX = maxX; this.minY = minY; @@ -169,19 +149,25 @@ public class TextBlock extends AbstractTextContainer { this.rotation = rotation; } + public TextBlock union(TextPositionSequence r) { + TextBlock union = this.copy(); union.add(r); return union; } + public TextBlock union(TextBlock r) { + TextBlock union = this.copy(); union.add(r); return union; } + public void add(TextBlock r) { + if (r.getMinX() < minX) { minX = r.getMinX(); } @@ -197,7 +183,9 @@ public class TextBlock extends AbstractTextContainer { sequences.addAll(r.getSequences()); } + public void add(TextPositionSequence r) { + if (r.getX1() < minX) { minX = r.getX1(); } @@ -212,15 +200,21 @@ public class TextBlock extends AbstractTextContainer { } } + public TextBlock copy() { + return new TextBlock(minX, maxX, minY, maxY, sequences, rotation); } + public void resize(float x1, float y1, float width, float height) { + set(x1, y1, x1 + width, y1 + height); } + public void set(float x1, float y1, float x2, float y2) { + this.minX = Math.min(x1, x2); this.maxX = Math.max(x1, x2); this.minY = Math.min(y1, y2); @@ -246,6 +240,7 @@ public class TextBlock extends AbstractTextContainer { } + @Override @JsonIgnore @JsonAttribute(ignore = true) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java index 5ff32363..a055c80a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java @@ -2,8 +2,6 @@ package com.iqser.red.service.redaction.v1.server.classification.service; import static java.util.stream.Collectors.toSet; -import java.awt.geom.Line2D; -import java.awt.geom.Point2D; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; @@ -34,8 +32,7 @@ public class BlockificationService { static final float THRESHOLD = 1f; - public Page blockify(List textPositions, List horizontalRulingLines, - List verticalRulingLines, int rotation, PDRectangle cropbox) { + public Page blockify(List textPositions, List horizontalRulingLines, List verticalRulingLines, int rotation, PDRectangle cropbox) { // sortRotatedSequences(textPositions); @@ -49,17 +46,42 @@ public class BlockificationService { Float splitX1 = null; for (TextPositionSequence word : textPositions) { - boolean lineSeparation = word.getY1() - maxY > word.getHeight() * 1.25; + boolean lineSeparation = word.getY1() - maxY > word.getHeight() * 1.25; boolean startFromTop = prev != null && word.getY1() < prev.getY1() - prev.getTextHeight(); boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1(); boolean xIsBeforeFirstX = prev != null && word.getX1() < minX; boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1(); - boolean splittedByRuling = - isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) || - isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) + boolean splittedByRuling = isSplittedByRuling(maxX, + minY, + word.getX1(), + word.getY1(), + verticalRulingLines, + word.getDir().getDegrees(), + cropbox.getWidth(), + cropbox.getHeight()) || isSplittedByRuling(minX, + minY, + word.getX1(), + word.getY2(), + horizontalRulingLines, + word.getDir().getDegrees(), + cropbox.getWidth(), + cropbox.getHeight()) - || isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) - || isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()); + || isSplittedByRuling(maxX, + minY, + word.getX1(), + word.getY1(), + horizontalRulingLines, + word.getDir().getDegrees(), + cropbox.getWidth(), + cropbox.getHeight()) || isSplittedByRuling(minX, + minY, + word.getX1(), + word.getY2(), + verticalRulingLines, + word.getDir().getDegrees(), + cropbox.getWidth(), + cropbox.getHeight()); boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir()); if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) { @@ -122,8 +144,7 @@ public class BlockificationService { TextBlock block = (TextBlock) itty.next(); if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { - if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft - .getMinY()) { + if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) { previousLeft.add(block); itty.remove(); continue; @@ -131,8 +152,7 @@ public class BlockificationService { } if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) { - if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight - .getMinY()) { + if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) { previousRight.add(block); itty.remove(); continue; @@ -151,10 +171,8 @@ public class BlockificationService { while (itty.hasNext()) { TextBlock block = (TextBlock) itty.next(); - if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() - .equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous - .getOrientation() - .equals(Orientation.LEFT) && block.getOrientation() + if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), + previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) { previous.add(block); itty.remove(); @@ -193,12 +211,10 @@ public class BlockificationService { styleFrequencyCounter.add(wordBlock.getFontStyle()); if (textBlock == null) { - textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock - .getRotation()); + textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation()); } else { TextBlock spatialEntity = textBlock.union(wordBlock); - textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity - .getHeight()); + textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight()); } } @@ -211,19 +227,21 @@ public class BlockificationService { textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest()); } - if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences() - .stream() - .map(t -> round(t.getY1(), 3)) - .collect(toSet()) - .size() == 1) { + if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) { textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1)); } return textBlock; } - private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, - List rulingLines, float rotation, float pageWidth, float pageHeight) { + private boolean isSplittedByRuling(float previousX2, + float previousY1, + float currentX1, + float currentY1, + List rulingLines, + float rotation, + float pageWidth, + float pageHeight) { for (Ruling ruling : rulingLines) { var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight); @@ -235,13 +253,7 @@ public class BlockificationService { } - - - - - - public Rectangle calculateBodyTextFrame(List pages, FloatFrequencyCounter documentFontSizeCounter, - boolean landscape) { + public Rectangle calculateBodyTextFrame(List pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) { float minX = 10000; float maxX = -100; @@ -270,7 +282,7 @@ public class BlockificationService { if (documentFontSizeCounter.getMostPopular() != null) { if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) { - if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){ + if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) { if (textBlock.getY1() < minX) { minX = textBlock.getY1(); } @@ -310,7 +322,7 @@ public class BlockificationService { continue; } for (TextBlock textBlock : cell.getTextBlocks()) { - if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){ + if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) { if (textBlock.getY1() < minX) { minX = textBlock.getMinY(); } @@ -342,16 +354,13 @@ public class BlockificationService { } } } - System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape()+ " MinX: " + minX + " MaxX: " + maxX + " MinY: " +minY + " MaxY: " + maxY); + System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape() + " MinX: " + minX + " MaxX: " + maxX + " MinY: " + minY + " MaxY: " + maxY); } return new Rectangle(new Point(minX, minY), maxX - minX, maxY - minY, 0); } - - - private double round(float value, int decimalPoints) { var d = Math.pow(10, decimalPoints); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java index 41a72734..56f09335 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java @@ -1,5 +1,10 @@ package com.iqser.red.service.redaction.v1.server.classification.service; +import java.util.List; +import java.util.regex.Pattern; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; @@ -7,12 +12,9 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.regex.Pattern; @Slf4j @Service @@ -34,24 +36,21 @@ public class ClassificationService { for (Page page : document.getPages()) { Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame; + if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) { + btf = new Rectangle(new Point(btf.getTopLeft().getY(), page.getCropBox().getHeight() - btf.getTopLeft().getX() - btf.getWidth()), + btf.getHeight(), + btf.getWidth(), + 0); + } else if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) { + btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0); + } else if (page.getRotation() == 180) { + btf = new Rectangle(new Point(btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() - btf.getHeight()), + btf.getWidth(), + btf.getHeight(), + 0); + } - if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) { - btf = new Rectangle(new Point(btf.getTopLeft().getY(),page.getCropBox().getHeight() - btf.getTopLeft().getX() -btf.getWidth()), btf.getHeight(), btf.getWidth(), 0); - } - else if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) { - btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0); - } - - else if(page.getRotation() == 180){ - btf = new Rectangle(new Point( btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() -btf.getHeight()), btf.getWidth(), btf.getHeight(), 0); - } - - - - page.setBodyTextFrame(btf); - - - + page.setBodyTextFrame(btf); classifyPage(btf, page, document, headlineFontSizes); } @@ -68,10 +67,9 @@ public class ClassificationService { } - public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, - List headlineFontSizes) { + public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List headlineFontSizes) { - System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() +" B " + textBlock.getSequences().get(0).getDir()); + System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() + " B " + textBlock.getSequences().get(0).getDir()); // if (document.getFontSizeCounter().getMostPopular() == null) { // textBlock.setClassification("Other"); @@ -91,19 +89,21 @@ public class ClassificationService { // .getMostPopular())) { textBlock.setClassification("Footer"); - } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils - .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter() - .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter() - .getMostPopular() || page.getTextBlocks().size() == 1)) { + } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, + textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, + document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks() + .size() == 1)) { if (!Pattern.matches("[0-9]+", textBlock.toString())) { textBlock.setClassification("Title"); } - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document - .getFontSizeCounter() - .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle() - .equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document - .getFontSizeCounter() - .getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { + } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() + .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() + .getCountPerValue() + .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences() + .get(0) + .getTextPositions() + .get(0) + .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { for (int i = 1; i <= headlineFontSizes.size(); i++) { if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { @@ -111,28 +111,25 @@ public class ClassificationService { document.setHeadlines(true); } } - } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText() - .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle() - .equals("bold") && !document.getFontStyleCounter() + } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, + textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() .getMostPopular() - .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { + .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() + .get(0) + .getTextPositions() + .get(0) + .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); document.setHeadlines(true); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document - .getFontSizeCounter() - .getMostPopular() && textBlock.getMostPopularWordStyle() - .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { + } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() + .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { textBlock.setClassification("TextBlock Bold"); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle() - .equals(document.getFontStyleCounter() - .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() - .getMostPopular()) { + .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) { textBlock.setClassification("TextBlock"); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document - .getFontSizeCounter() - .getMostPopular() && textBlock.getMostPopularWordStyle() - .equals("italic") && !document.getFontStyleCounter() + } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() + .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter() .getMostPopular() .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { textBlock.setClassification("TextBlock Italic"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java index b24d65fb..4a2b7c14 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java @@ -12,14 +12,15 @@ import lombok.experimental.UtilityClass; @UtilityClass public class CoordSystemHelper { - public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) { var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight); - var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() , rotation, pageWidth, pageHeight); + var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rotation, pageWidth, pageHeight); - - return new Rectangle(new Point((float)topLeft.getX(),(float) topLeft.getY()), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()), 0); + return new Rectangle(new Point((float) topLeft.getX(), (float) topLeft.getY()), + (float) (bottomRight.getX() - topLeft.getX()), + (float) (bottomRight.getY() - topLeft.getY()), + 0); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java index 95aede54..932e99e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java @@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.classification.utils; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; + import lombok.experimental.UtilityClass; @UtilityClass @SuppressWarnings("all") public class PositionUtils { - public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) { //TODO Currently this is not working for rotated pages. @@ -19,10 +19,9 @@ public class PositionUtils { double threshold = textBlock.getMostPopularWordHeight() * 3; - if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && - textBlock.getMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth() && - textBlock.getMinY() + threshold > btf.getTopLeft().getY() && - textBlock.getMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) { + if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && textBlock.getMaxX() - threshold < btf.getTopLeft() + .getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getTopLeft().getY() && textBlock.getMaxY() - threshold < btf.getTopLeft() + .getY() + btf.getHeight()) { return true; } else { return false; @@ -41,7 +40,7 @@ public class PositionUtils { return true; } - if(rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()){ + if (rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()) { return true; } @@ -60,7 +59,6 @@ public class PositionUtils { public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) { - if (btf == null || textBlock == null) { return false; } @@ -104,11 +102,14 @@ public class PositionUtils { public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) { + return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight; } public Float getApproxLineCount(TextBlock textBlock) { + return textBlock.getHeight() / textBlock.getMostPopularWordHeight(); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java index f51ec205..446257d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java @@ -147,7 +147,6 @@ public class TextPositionSequence implements CharSequence { } - @JsonIgnore @JsonAttribute(ignore = true) public float getRotationAdjustedY() { @@ -160,8 +159,7 @@ public class TextPositionSequence implements CharSequence { @JsonAttribute(ignore = true) public float getX1() { - - return textPositions.get(0).getXDirAdj(); + return textPositions.get(0).getXDirAdj(); } @@ -170,25 +168,16 @@ public class TextPositionSequence implements CharSequence { @JsonAttribute(ignore = true) public float getX2() { - - return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING; + return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING; } - - - @JsonIgnore @JsonAttribute(ignore = true) public float getY1() { - - - - - - return textPositions.get(0).getYDirAdj() - getTextHeight(); + return textPositions.get(0).getYDirAdj() - getTextHeight(); } @@ -197,15 +186,11 @@ public class TextPositionSequence implements CharSequence { @JsonAttribute(ignore = true) public float getY2() { - - - return textPositions.get(0).getYDirAdj(); } - @JsonIgnore @JsonAttribute(ignore = true) public float getTextHeight() { @@ -307,7 +292,10 @@ public class TextPositionSequence implements CharSequence { topRight = transform.transform(topRight, null); return new Rectangle( // - new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), page); + new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), + (float) (topRight.getX() - bottomLeft.getX()), + (float) (topRight.getY() - bottomLeft.getY()), + page); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index 0cf72b50..2cd85d50 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -101,18 +101,20 @@ public class PdfSegmentationService { PDRectangle pdr = pdPage.getMediaBox(); int rotation = pdPage.getRotation(); - boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || - pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270); + boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270); PDRectangle cropbox = pdPage.getCropBox(); - CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight()); + CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), + stripper.getRulings(), + stripper.getMinCharWidth(), + stripper.getMaxCharHeight()); Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox); - float cropboxArea = cropbox.getHeight() * cropbox.getWidth(); page.setCropBoxArea(cropboxArea); - System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" +cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox().getWidth() + " h:" +pdPage.getMediaBox().getHeight()); + System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" + cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox() + .getWidth() + " h:" + pdPage.getMediaBox().getHeight()); page.setRotation(rotation); page.setLandscape(isLandscape); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java index 4dc3f7ef..ae05eaba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java @@ -31,31 +31,41 @@ public abstract class AbstractTextContainer { @JsonIgnore private Orientation orientation = Orientation.NONE; + public abstract String getText(); public boolean containsBlock(TextBlock other) { + return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2(); } public boolean contains(AbstractTextContainer other) { + return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY; } + public boolean contains(Rectangle other) { - return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight(); + + return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft() + .getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight(); } + @JsonIgnore @JsonAttribute(ignore = true) public float getHeight() { + return maxY - minY; } + @JsonIgnore @JsonAttribute(ignore = true) public float getWidth() { + return maxX - minX; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java index 8ec3e473..ad4a82b4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java @@ -1,14 +1,27 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.service; +import java.awt.geom.Point2D; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.tableextraction.model.*; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils; -import org.springframework.stereotype.Service; - -import java.awt.geom.Point2D; -import java.util.*; -import java.util.stream.Collectors; @Service public class TableExtractionService { @@ -52,6 +65,7 @@ public class TableExtractionService { return rv; }; + public void extractTables(CleanRulings cleanRulings, Page page) { List cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical()); @@ -72,9 +86,7 @@ public class TableExtractionService { cells = new ArrayList<>(new HashSet<>(cells)); Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER); - List spreadsheetAreas = findSpreadsheetsFromCells(cells).stream() - .filter(r -> r.getWidth() > 0f && r.getHeight() > 0f) - .collect(Collectors.toList()); + List spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).collect(Collectors.toList()); List tables = new ArrayList<>(); for (Rectangle area : spreadsheetAreas) { @@ -106,6 +118,7 @@ public class TableExtractionService { page.getTextBlocks().removeAll(toBeRemoved); } + public List findCells(List horizontalRulingLines, List verticalRulingLines) { List cellsFound = new ArrayList<>(); @@ -142,8 +155,8 @@ public class TableExtractionService { continue; } Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY()); - if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints - .get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) { + if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals( + intersectionPoints.get(yPoint)[1])) { cellsFound.add(new Cell(topLeft, btmRight)); break outer; } @@ -158,6 +171,7 @@ public class TableExtractionService { return cellsFound; } + private List findSpreadsheetsFromCells(List cells) { // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon List rectangles = new ArrayList<>(); @@ -257,8 +271,10 @@ public class TableExtractionService { return rectangles; } + private enum Direction { - HORIZONTAL, VERTICAL + HORIZONTAL, + VERTICAL } static class PolygonVertex { @@ -297,8 +313,7 @@ public class TableExtractionService { @Override public String toString() { - return String.format("%s[point=%s,direction=%s]", this.getClass() - .getName(), this.point.toString(), this.direction.toString()); + return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java index 49b5f89f..dd6464b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java @@ -1,5 +1,15 @@ package com.iqser.red.service.redaction.v1.server.visualization.service; +import java.awt.Color; +import java.io.IOException; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.springframework.stereotype.Service; + import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; @@ -7,24 +17,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.PDPageContentStream; -import org.apache.pdfbox.pdmodel.font.PDType1Font; -import org.springframework.stereotype.Service; - -import java.awt.Color; -import java.io.IOException; -import java.util.List; @Slf4j @Service @RequiredArgsConstructor public class PdfVisualisationService { - public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException { for (int page = 1; page <= document.getNumberOfPages(); page++) { @@ -66,7 +67,6 @@ public class PdfVisualisationService { PDPage pdPage = document.getPage(page - 1); PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true); - for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) { if (textBlock == null) { continue; @@ -84,7 +84,10 @@ public class PdfVisualisationService { // // contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height); - contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), analyzedPage.getBodyTextFrame().getTopLeft().getY(), analyzedPage.getBodyTextFrame().getWidth(), analyzedPage.getBodyTextFrame().getHeight()); + contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), + analyzedPage.getBodyTextFrame().getTopLeft().getY(), + analyzedPage.getBodyTextFrame().getWidth(), + analyzedPage.getBodyTextFrame().getHeight()); contentStream.stroke(); @@ -111,7 +114,6 @@ public class PdfVisualisationService { contentStream.endText(); - // contentStream.setNonStrokingColor(Color.BLUE); // contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f); // @@ -140,6 +142,7 @@ public class PdfVisualisationService { private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException { + for (List row : table.getRows()) { for (Cell cell : row) { @@ -170,4 +173,5 @@ public class PdfVisualisationService { contentStream.endText(); } } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 181907f8..fc93b818 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -183,8 +183,7 @@ public class RedactionIntegrationTest { KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8)); - kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources() - .newInputStreamResource(input)); + kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); KieModule kieModule = kieBuilder.getKieModule(); @@ -245,16 +244,25 @@ public class RedactionIntegrationTest { private void mockDictionaryCalls(Long version) { - when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, false)); + when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, + false)); when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ADDRESS, false)); when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(AUTHOR, false)); when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SPONSOR, false)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + NO_REDACTION_INDICATOR, + false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + REDACTION_INDICATOR, + false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT, false)); - when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION, false)); - when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT, + false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + PUBLISHED_INFORMATION, + false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD, + false)); when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY, false)); when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE, false)); @@ -262,9 +270,14 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO, false)); when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE, false)); when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA, false)); - when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE, false)); - when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS, true)); - when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION, true)); + when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE, + false)); + when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + DOSSIER_REDACTIONS, + true)); + when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + IMPORTED_REDACTION, + true)); } @@ -312,10 +325,7 @@ public class RedactionIntegrationTest { dictionary.get(AUTHOR).add("Drinking water"); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated3.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -345,16 +355,12 @@ public class RedactionIntegrationTest { var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getRedactionLogEntry() - .stream() - .map(RedactionLogEntry::getValue) - .collect(Collectors.toList()); + var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList()); assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname"); } - @Test public void titleExtraction() throws IOException { @@ -367,10 +373,7 @@ public class RedactionIntegrationTest { var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; @@ -381,8 +384,6 @@ public class RedactionIntegrationTest { } - - @Test @Ignore @SneakyThrows @@ -417,18 +418,10 @@ public class RedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); - var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry() - .stream() - .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) - .findAny() - .get(); + var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse(); - var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry() - .stream() - .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) - .findAny() - .get(); + var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue(); } @@ -526,8 +519,7 @@ public class RedactionIntegrationTest { continue loop; } if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) { - String value = sectionText.getText() - .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); + String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { correctFound++; } else { @@ -573,10 +565,7 @@ public class RedactionIntegrationTest { end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); @@ -627,19 +616,11 @@ public class RedactionIntegrationTest { var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var changes = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getValue() != null && entry.getValue().equals("report")) - .findFirst() - .get() - .getChanges(); + var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges(); assertThat(changes.size()).isEqualTo(2); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); @@ -690,8 +671,7 @@ public class RedactionIntegrationTest { continue loop; } if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) { - String value = sectionText.getText() - .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); + String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { correctFound++; } else { @@ -740,10 +720,7 @@ public class RedactionIntegrationTest { end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); @@ -840,10 +817,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -867,10 +841,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -894,10 +865,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -918,10 +886,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -943,10 +908,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -957,8 +919,10 @@ public class RedactionIntegrationTest { System.out.println("numberOfPages: " + result.getNumberOfPages()); } + @Test public void testRotationsSimple() throws IOException { + loadOnlyDictionaryForSimpleFile(); mockDictionaryCalls(null); mockDictionaryCalls(0L); @@ -970,10 +934,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/AnnotatedSimple.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -995,10 +956,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -1014,7 +972,6 @@ public class RedactionIntegrationTest { } - @Test public void testFindDictionaryEntryInResizedEntryPosition() throws IOException { @@ -1034,7 +991,6 @@ public class RedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.getResizeRedactions() .add(ManualResizeRedaction.builder() .annotationId("ca2b437e2480a4b5966cb8386020d454") @@ -1084,16 +1040,8 @@ public class RedactionIntegrationTest { String manualAddId = UUID.randomUUID().toString(); - Comment comment = Comment.builder() - .date(OffsetDateTime.now()) - .user("TEST_USER") - .text("This is a comment test") - .build(); - manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() - .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") - .fileId("fileId") - .status(AnnotationStatus.DECLINED) - .build())); + Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build(); + manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") @@ -1112,19 +1060,8 @@ public class RedactionIntegrationTest { manualRedactionEntry.setType("name"); manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setReason("Manual Redaction"); - manualRedactionEntry.setPositions(List.of(Rectangle.builder() - .topLeftX(375.61096f) - .topLeftY(241.282f) - .width(7.648041f) - .height(43.72262f) - .page(1) - .build(), Rectangle.builder() - .topLeftX(384.83517f) - .topLeftY(241.282f) - .width(7.648041f) - .height(17.043358f) - .page(1) - .build())); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), + Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); AnalyzeRequest request = prepareStorage(pdfFile); request.setManualRedactions(manualRedactions); @@ -1132,11 +1069,7 @@ public class RedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); - manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() - .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") - .fileId("fileId") - .status(AnnotationStatus.APPROVED) - .build())); + manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.APPROVED).build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") @@ -1148,10 +1081,7 @@ public class RedactionIntegrationTest { var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -1183,6 +1113,7 @@ public class RedactionIntegrationTest { } } + @Test public void classificationTestWithCvTableService() throws IOException { @@ -1192,10 +1123,10 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile); RedactionRequest redactionRequest = RedactionRequest.builder() - .dossierId(request.getDossierId()) - .fileId(request.getFileId()) - .dossierTemplateId(request.getDossierTemplateId()) - .build(); + .dossierId(request.getDossierId()) + .fileId(request.getFileId()) + .dossierTemplateId(request.getDossierTemplateId()) + .build(); RedactionResult result = redactionController.classify(redactionRequest); @@ -1295,10 +1226,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -1320,11 +1248,7 @@ public class RedactionIntegrationTest { String manualAddId = UUID.randomUUID().toString(); - Comment comment = Comment.builder() - .date(OffsetDateTime.now()) - .user("TEST_USER") - .text("This is a comment test") - .build(); + Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build(); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") .fileId("fileId") @@ -1354,19 +1278,8 @@ public class RedactionIntegrationTest { manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); manualRedactionEntry.setRequestDate(OffsetDateTime.now()); - manualRedactionEntry.setPositions(List.of(Rectangle.builder() - .topLeftX(375.61096f) - .topLeftY(241.282f) - .width(7.648041f) - .height(43.72262f) - .page(1) - .build(), Rectangle.builder() - .topLeftX(384.83517f) - .topLeftY(241.282f) - .width(7.648041f) - .height(17.043358f) - .page(1) - .build())); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), + Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); // manualRedactions.getEntriesToAdd().add(manualRedactionEntry); @@ -1397,31 +1310,10 @@ public class RedactionIntegrationTest { .processedDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now()) .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland") - .positions(List.of(Rectangle.builder() - .topLeftX(289.44595f) - .topLeftY(327.567f) - .width(7.648041f) - .height(82.51475f) - .page(1) - .build(), Rectangle.builder() - .topLeftX(298.67056f) - .topLeftY(327.567f) - .width(7.648041f) - .height(75.32377f) - .page(1) - .build(), Rectangle.builder() - .topLeftX(307.89517f) - .topLeftY(327.567f) - .width(7.648041f) - .height(61.670967f) - .page(1) - .build(), Rectangle.builder() - .topLeftX(316.99985f) - .topLeftY(327.567f) - .width(7.648041f) - .height(38.104286f) - .page(1) - .build())) + .positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(), + Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(), + Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(), + Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build())) .status(AnnotationStatus.APPROVED) .build())); @@ -1451,10 +1343,7 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -1481,13 +1370,7 @@ public class RedactionIntegrationTest { manualRedactionEntry.setType("CBI_author"); manualRedactionEntry.setValue("rabbits"); manualRedactionEntry.setReason("Manual Redaction"); - manualRedactionEntry.setPositions(List.of(Rectangle.builder() - .topLeftX(70.944f) - .topLeftY(670.1595f) - .width(30.07296f) - .height(10.048125f) - .page(1) - .build())); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build())); ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry(); manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString()); @@ -1496,13 +1379,7 @@ public class RedactionIntegrationTest { manualRedactionEntry2.setType("CBI_author"); manualRedactionEntry2.setValue("rabbits"); manualRedactionEntry2.setReason("Manual Redaction"); - manualRedactionEntry2.setPositions(List.of(Rectangle.builder() - .topLeftX(470.5204f) - .topLeftY(746.1195f) - .width(29.96256f) - .height(10.048125f) - .page(1) - .build())); + manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build())); ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry(); manualRedactionEntry3.setAnnotationId(UUID.randomUUID().toString()); @@ -1511,13 +1388,7 @@ public class RedactionIntegrationTest { manualRedactionEntry3.setType("CBI_author"); manualRedactionEntry3.setValue("AOEL"); manualRedactionEntry3.setReason("Manual Redaction"); - manualRedactionEntry3.setPositions(List.of(Rectangle.builder() - .topLeftX(355.53775f) - .topLeftY(266.1895f) - .width(29.32224f) - .height(10.048125f) - .page(1) - .build())); + manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build())); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); @@ -1539,8 +1410,7 @@ public class RedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions) - .getManualRedactions(); + var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions).getManualRedactions(); surroundingTextResult.getEntriesToAdd().forEach(addEntry -> { assertThat(addEntry.getTextAfter()).isNotEmpty(); }); @@ -1551,112 +1421,54 @@ public class RedactionIntegrationTest { private void loadDictionaryForTest() { dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_author.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/vertebrate.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_address.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/hint_only.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/must_redact.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/published_information.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/test_method.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/purity.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); } + private void loadOnlyDictionaryForSimpleFile() { + dictionary.clear(); dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); } @@ -1851,6 +1663,7 @@ public class RedactionIntegrationTest { .build(); } + private String cleanDictionaryEntry(String entry) { return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); @@ -1858,16 +1671,14 @@ public class RedactionIntegrationTest { private List toDictionaryEntry(List entries) { - if (entries == null) + + if (entries == null) { entries = Collections.emptyList(); + } List dictionaryEntries = new ArrayList<>(); entries.forEach(entry -> { - dictionaryEntries.add(DictionaryEntry.builder() - .value(entry) - .version(reanlysisVersions.getOrDefault(entry, 0L)) - .deleted(deleted.contains(entry)) - .build()); + dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build()); }); return dictionaryEntries; } @@ -1880,17 +1691,15 @@ public class RedactionIntegrationTest { ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json"); AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream()); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), + importedRedactions.getInputStream()); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); redactionLog.getRedactionLogEntry().forEach(entry -> { if (entry.getValue() == null) { @@ -1928,20 +1737,14 @@ public class RedactionIntegrationTest { analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); } var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getRedactionLogEntry() - .stream() - .map(RedactionLogEntry::getValue) - .collect(Collectors.toList()); + var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList()); assertThat(values).contains("Mrs. Robinson"); assertThat(values).contains("Mr. Bojangles"); @@ -1973,6 +1776,7 @@ public class RedactionIntegrationTest { return prepareStorage(file, "files/cv_service_empty_response.json"); } + @SneakyThrows private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {