RED-5381: Use new codestyle
This commit is contained in:
deiflaender
parent
87296af54e
commit
ad265f9256
@ -1,17 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class Page {
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
@ -7,14 +10,12 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Data
|
||||
@ -46,20 +47,24 @@ public class TextBlock extends AbstractTextContainer {
|
||||
@JsonIgnore
|
||||
private float highestFontSize;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
private String classification;
|
||||
|
||||
|
||||
private TextDirection getDir() {
|
||||
|
||||
return sequences.get(0).getDir();
|
||||
}
|
||||
|
||||
|
||||
private float getPageHeight() {
|
||||
|
||||
return sequences.get(0).getPageHeight();
|
||||
}
|
||||
|
||||
|
||||
private float getPageWidth() {
|
||||
|
||||
return sequences.get(0).getPageWidth();
|
||||
}
|
||||
|
||||
@ -70,16 +75,13 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minY;
|
||||
}
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - maxX;
|
||||
|
||||
}
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
|
||||
return getPageWidth() - maxY;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return minX;
|
||||
}
|
||||
}
|
||||
@ -91,44 +93,30 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxY;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - minX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageWidth() - minY;
|
||||
|
||||
}
|
||||
|
||||
else {
|
||||
} else {
|
||||
return maxX;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY1() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return maxY;
|
||||
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - maxX;
|
||||
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return getPageHeight() - maxY;
|
||||
}
|
||||
}
|
||||
@ -140,27 +128,19 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 180 ) {
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
|
||||
return minY;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270 ) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - minX;
|
||||
}
|
||||
|
||||
else {
|
||||
} else {
|
||||
return getPageHeight() - minY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
|
||||
|
||||
this.minX = minX;
|
||||
this.maxX = maxX;
|
||||
this.minY = minY;
|
||||
@ -169,19 +149,25 @@ public class TextBlock extends AbstractTextContainer {
|
||||
this.rotation = rotation;
|
||||
}
|
||||
|
||||
|
||||
public TextBlock union(TextPositionSequence r) {
|
||||
|
||||
TextBlock union = this.copy();
|
||||
union.add(r);
|
||||
return union;
|
||||
}
|
||||
|
||||
|
||||
public TextBlock union(TextBlock r) {
|
||||
|
||||
TextBlock union = this.copy();
|
||||
union.add(r);
|
||||
return union;
|
||||
}
|
||||
|
||||
|
||||
public void add(TextBlock r) {
|
||||
|
||||
if (r.getMinX() < minX) {
|
||||
minX = r.getMinX();
|
||||
}
|
||||
@ -197,7 +183,9 @@ public class TextBlock extends AbstractTextContainer {
|
||||
sequences.addAll(r.getSequences());
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPositionSequence r) {
|
||||
|
||||
if (r.getX1() < minX) {
|
||||
minX = r.getX1();
|
||||
}
|
||||
@ -212,15 +200,21 @@ public class TextBlock extends AbstractTextContainer {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public TextBlock copy() {
|
||||
|
||||
return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
|
||||
}
|
||||
|
||||
|
||||
public void resize(float x1, float y1, float width, float height) {
|
||||
|
||||
set(x1, y1, x1 + width, y1 + height);
|
||||
}
|
||||
|
||||
|
||||
public void set(float x1, float y1, float x2, float y2) {
|
||||
|
||||
this.minX = Math.min(x1, x2);
|
||||
this.maxX = Math.max(x1, x2);
|
||||
this.minY = Math.min(y1, y2);
|
||||
@ -246,6 +240,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
|
||||
@ -2,8 +2,6 @@ package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import static java.util.stream.Collectors.toSet;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
@ -34,8 +32,7 @@ public class BlockificationService {
|
||||
static final float THRESHOLD = 1f;
|
||||
|
||||
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
|
||||
List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
|
||||
|
||||
// sortRotatedSequences(textPositions);
|
||||
|
||||
@ -54,12 +51,37 @@ public class BlockificationService {
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
|
||||
boolean xIsBeforeFirstX = prev != null && word.getX1() < minX;
|
||||
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
|
||||
boolean splittedByRuling =
|
||||
isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) ||
|
||||
isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight())
|
||||
boolean splittedByRuling = isSplittedByRuling(maxX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY1(),
|
||||
verticalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight()) || isSplittedByRuling(minX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY2(),
|
||||
horizontalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight())
|
||||
|
||||
|| isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight())
|
||||
|| isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight());
|
||||
|| isSplittedByRuling(maxX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY1(),
|
||||
horizontalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight()) || isSplittedByRuling(minX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY2(),
|
||||
verticalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight());
|
||||
boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) {
|
||||
@ -122,8 +144,7 @@ public class BlockificationService {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft
|
||||
.getMinY()) {
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
|
||||
previousLeft.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
@ -131,8 +152,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight
|
||||
.getMinY()) {
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
|
||||
previousRight.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
@ -151,10 +171,8 @@ public class BlockificationService {
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
|
||||
.equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous
|
||||
.getOrientation()
|
||||
.equals(Orientation.LEFT) && block.getOrientation()
|
||||
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
|
||||
previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
|
||||
.equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
|
||||
previous.add(block);
|
||||
itty.remove();
|
||||
@ -193,12 +211,10 @@ public class BlockificationService {
|
||||
styleFrequencyCounter.add(wordBlock.getFontStyle());
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
|
||||
.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
|
||||
} else {
|
||||
TextBlock spatialEntity = textBlock.union(wordBlock);
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
|
||||
.getHeight());
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
|
||||
}
|
||||
}
|
||||
|
||||
@ -211,19 +227,21 @@ public class BlockificationService {
|
||||
textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
|
||||
}
|
||||
|
||||
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences()
|
||||
.stream()
|
||||
.map(t -> round(t.getY1(), 3))
|
||||
.collect(toSet())
|
||||
.size() == 1) {
|
||||
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) {
|
||||
textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1));
|
||||
}
|
||||
return textBlock;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
|
||||
List<Ruling> rulingLines, float rotation, float pageWidth, float pageHeight) {
|
||||
private boolean isSplittedByRuling(float previousX2,
|
||||
float previousY1,
|
||||
float currentX1,
|
||||
float currentY1,
|
||||
List<Ruling> rulingLines,
|
||||
float rotation,
|
||||
float pageWidth,
|
||||
float pageHeight) {
|
||||
|
||||
for (Ruling ruling : rulingLines) {
|
||||
var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight);
|
||||
@ -235,13 +253,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
|
||||
boolean landscape) {
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
@ -349,9 +361,6 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private double round(float value, int decimalPoints) {
|
||||
|
||||
var d = Math.pow(10, decimalPoints);
|
||||
|
||||
@ -1,5 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
@ -7,12 +12,9 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -34,25 +36,22 @@ public class ClassificationService {
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
|
||||
|
||||
if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(),page.getCropBox().getHeight() - btf.getTopLeft().getX() -btf.getWidth()), btf.getHeight(), btf.getWidth(), 0);
|
||||
}
|
||||
else if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(), page.getCropBox().getHeight() - btf.getTopLeft().getX() - btf.getWidth()),
|
||||
btf.getHeight(),
|
||||
btf.getWidth(),
|
||||
0);
|
||||
} else if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0);
|
||||
} else if (page.getRotation() == 180) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() - btf.getHeight()),
|
||||
btf.getWidth(),
|
||||
btf.getHeight(),
|
||||
0);
|
||||
}
|
||||
|
||||
else if(page.getRotation() == 180){
|
||||
btf = new Rectangle(new Point( btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() -btf.getHeight()), btf.getWidth(), btf.getHeight(), 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
page.setBodyTextFrame(btf);
|
||||
|
||||
|
||||
|
||||
|
||||
classifyPage(btf, page, document, headlineFontSizes);
|
||||
}
|
||||
}
|
||||
@ -68,8 +67,7 @@ public class ClassificationService {
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() + " B " + textBlock.getSequences().get(0).getDir());
|
||||
|
||||
@ -91,19 +89,21 @@ public class ClassificationService {
|
||||
// .getMostPopular()))
|
||||
{
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame,
|
||||
textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
|
||||
document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
|
||||
.size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
|
||||
.getCountPerValue()
|
||||
.containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@ -111,28 +111,25 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
|
||||
textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
.equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
|
||||
@ -12,14 +12,15 @@ import lombok.experimental.UtilityClass;
|
||||
@UtilityClass
|
||||
public class CoordSystemHelper {
|
||||
|
||||
|
||||
public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) {
|
||||
|
||||
var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight);
|
||||
var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rotation, pageWidth, pageHeight);
|
||||
|
||||
|
||||
return new Rectangle(new Point((float)topLeft.getX(),(float) topLeft.getY()), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()), 0);
|
||||
return new Rectangle(new Point((float) topLeft.getX(), (float) topLeft.getY()),
|
||||
(float) (bottomRight.getX() - topLeft.getX()),
|
||||
(float) (bottomRight.getY() - topLeft.getY()),
|
||||
0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
@SuppressWarnings("all")
|
||||
public class PositionUtils {
|
||||
|
||||
|
||||
public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
//TODO Currently this is not working for rotated pages.
|
||||
@ -19,10 +19,9 @@ public class PositionUtils {
|
||||
|
||||
double threshold = textBlock.getMostPopularWordHeight() * 3;
|
||||
|
||||
if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() &&
|
||||
textBlock.getMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth() &&
|
||||
textBlock.getMinY() + threshold > btf.getTopLeft().getY() &&
|
||||
textBlock.getMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
|
||||
if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && textBlock.getMaxX() - threshold < btf.getTopLeft()
|
||||
.getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getTopLeft().getY() && textBlock.getMaxY() - threshold < btf.getTopLeft()
|
||||
.getY() + btf.getHeight()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@ -60,7 +59,6 @@ public class PositionUtils {
|
||||
|
||||
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
|
||||
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
@ -104,11 +102,14 @@ public class PositionUtils {
|
||||
|
||||
|
||||
public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
|
||||
|
||||
return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
|
||||
}
|
||||
|
||||
|
||||
public Float getApproxLineCount(TextBlock textBlock) {
|
||||
|
||||
return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -147,7 +147,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getRotationAdjustedY() {
|
||||
@ -160,7 +159,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getX1() {
|
||||
|
||||
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
|
||||
}
|
||||
@ -170,24 +168,15 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getX2() {
|
||||
|
||||
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY1() {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
|
||||
}
|
||||
@ -197,15 +186,11 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY2() {
|
||||
|
||||
|
||||
|
||||
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getTextHeight() {
|
||||
@ -307,7 +292,10 @@ public class TextPositionSequence implements CharSequence {
|
||||
topRight = transform.transform(topRight, null);
|
||||
|
||||
return new Rectangle( //
|
||||
new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), page);
|
||||
new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()),
|
||||
(float) (topRight.getX() - bottomLeft.getX()),
|
||||
(float) (topRight.getY() - bottomLeft.getY()),
|
||||
page);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -101,18 +101,20 @@ public class PdfSegmentationService {
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) ||
|
||||
pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
|
||||
|
||||
PDRectangle cropbox = pdPage.getCropBox();
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
|
||||
stripper.getRulings(),
|
||||
stripper.getMinCharWidth(),
|
||||
stripper.getMaxCharHeight());
|
||||
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox);
|
||||
|
||||
|
||||
float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
|
||||
page.setCropBoxArea(cropboxArea);
|
||||
|
||||
System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" +cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox().getWidth() + " h:" +pdPage.getMediaBox().getHeight());
|
||||
System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" + cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox()
|
||||
.getWidth() + " h:" + pdPage.getMediaBox().getHeight());
|
||||
|
||||
page.setRotation(rotation);
|
||||
page.setLandscape(isLandscape);
|
||||
|
||||
@ -31,31 +31,41 @@ public abstract class AbstractTextContainer {
|
||||
@JsonIgnore
|
||||
private Orientation orientation = Orientation.NONE;
|
||||
|
||||
|
||||
public abstract String getText();
|
||||
|
||||
|
||||
public boolean containsBlock(TextBlock other) {
|
||||
|
||||
return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2();
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(AbstractTextContainer other) {
|
||||
|
||||
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
|
||||
.getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getHeight() {
|
||||
|
||||
return maxY - minY;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getWidth() {
|
||||
|
||||
return maxX - minX;
|
||||
}
|
||||
|
||||
|
||||
@ -1,14 +1,27 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.service;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class TableExtractionService {
|
||||
@ -52,6 +65,7 @@ public class TableExtractionService {
|
||||
return rv;
|
||||
};
|
||||
|
||||
|
||||
public void extractTables(CleanRulings cleanRulings, Page page) {
|
||||
|
||||
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
|
||||
@ -72,9 +86,7 @@ public class TableExtractionService {
|
||||
cells = new ArrayList<>(new HashSet<>(cells));
|
||||
Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
|
||||
|
||||
List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream()
|
||||
.filter(r -> r.getWidth() > 0f && r.getHeight() > 0f)
|
||||
.collect(Collectors.toList());
|
||||
List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).collect(Collectors.toList());
|
||||
|
||||
List<Table> tables = new ArrayList<>();
|
||||
for (Rectangle area : spreadsheetAreas) {
|
||||
@ -106,6 +118,7 @@ public class TableExtractionService {
|
||||
page.getTextBlocks().removeAll(toBeRemoved);
|
||||
}
|
||||
|
||||
|
||||
public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
|
||||
|
||||
List<Cell> cellsFound = new ArrayList<>();
|
||||
@ -142,8 +155,8 @@ public class TableExtractionService {
|
||||
continue;
|
||||
}
|
||||
Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
|
||||
if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints
|
||||
.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
|
||||
if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(
|
||||
intersectionPoints.get(yPoint)[1])) {
|
||||
cellsFound.add(new Cell(topLeft, btmRight));
|
||||
break outer;
|
||||
}
|
||||
@ -158,6 +171,7 @@ public class TableExtractionService {
|
||||
return cellsFound;
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
|
||||
// via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
@ -257,8 +271,10 @@ public class TableExtractionService {
|
||||
return rectangles;
|
||||
}
|
||||
|
||||
|
||||
private enum Direction {
|
||||
HORIZONTAL, VERTICAL
|
||||
HORIZONTAL,
|
||||
VERTICAL
|
||||
}
|
||||
|
||||
static class PolygonVertex {
|
||||
@ -297,8 +313,7 @@ public class TableExtractionService {
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return String.format("%s[point=%s,direction=%s]", this.getClass()
|
||||
.getName(), this.point.toString(), this.direction.toString());
|
||||
return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
@ -7,24 +17,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PdfVisualisationService {
|
||||
|
||||
|
||||
public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException {
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
@ -66,7 +67,6 @@ public class PdfVisualisationService {
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
|
||||
|
||||
for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) {
|
||||
if (textBlock == null) {
|
||||
continue;
|
||||
@ -84,7 +84,10 @@ public class PdfVisualisationService {
|
||||
//
|
||||
// contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height);
|
||||
|
||||
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), analyzedPage.getBodyTextFrame().getTopLeft().getY(), analyzedPage.getBodyTextFrame().getWidth(), analyzedPage.getBodyTextFrame().getHeight());
|
||||
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
|
||||
analyzedPage.getBodyTextFrame().getTopLeft().getY(),
|
||||
analyzedPage.getBodyTextFrame().getWidth(),
|
||||
analyzedPage.getBodyTextFrame().getHeight());
|
||||
|
||||
contentStream.stroke();
|
||||
|
||||
@ -111,7 +114,6 @@ public class PdfVisualisationService {
|
||||
|
||||
contentStream.endText();
|
||||
|
||||
|
||||
// contentStream.setNonStrokingColor(Color.BLUE);
|
||||
// contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
|
||||
//
|
||||
@ -140,6 +142,7 @@ public class PdfVisualisationService {
|
||||
|
||||
|
||||
private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
|
||||
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
@ -170,4 +173,5 @@ public class PdfVisualisationService {
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -183,8 +183,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
@ -245,16 +244,25 @@ public class RedactionIntegrationTest {
|
||||
|
||||
private void mockDictionaryCalls(Long version) {
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
NO_REDACTION_INDICATOR,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
REDACTION_INDICATOR,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT, false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION, false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
PUBLISHED_INFORMATION,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
|
||||
@ -262,9 +270,14 @@ public class RedactionIntegrationTest {
|
||||
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
|
||||
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
|
||||
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE, false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS, true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION, true));
|
||||
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
|
||||
false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
DOSSIER_REDACTIONS,
|
||||
true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
|
||||
IMPORTED_REDACTION,
|
||||
true));
|
||||
|
||||
}
|
||||
|
||||
@ -312,10 +325,7 @@ public class RedactionIntegrationTest {
|
||||
dictionary.get(AUTHOR).add("Drinking water");
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated3.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -345,16 +355,12 @@ public class RedactionIntegrationTest {
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var values = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.map(RedactionLogEntry::getValue)
|
||||
.collect(Collectors.toList());
|
||||
var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
|
||||
|
||||
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void titleExtraction() throws IOException {
|
||||
|
||||
@ -367,10 +373,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
@ -381,8 +384,6 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
@SneakyThrows
|
||||
@ -417,18 +418,10 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||
.findAny()
|
||||
.get();
|
||||
var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
|
||||
assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
|
||||
|
||||
var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||
.findAny()
|
||||
.get();
|
||||
var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
|
||||
assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
|
||||
}
|
||||
|
||||
@ -526,8 +519,7 @@ public class RedactionIntegrationTest {
|
||||
continue loop;
|
||||
}
|
||||
if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
|
||||
String value = sectionText.getText()
|
||||
.substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
|
||||
String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
|
||||
if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
|
||||
correctFound++;
|
||||
} else {
|
||||
@ -573,10 +565,7 @@ public class RedactionIntegrationTest {
|
||||
end = System.currentTimeMillis();
|
||||
System.out.println("reanalysis analysis duration: " + (end - start));
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -627,19 +616,11 @@ public class RedactionIntegrationTest {
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var changes = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
|
||||
.findFirst()
|
||||
.get()
|
||||
.getChanges();
|
||||
var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
|
||||
|
||||
assertThat(changes.size()).isEqualTo(2);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -690,8 +671,7 @@ public class RedactionIntegrationTest {
|
||||
continue loop;
|
||||
}
|
||||
if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
|
||||
String value = sectionText.getText()
|
||||
.substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
|
||||
String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
|
||||
if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
|
||||
correctFound++;
|
||||
} else {
|
||||
@ -740,10 +720,7 @@ public class RedactionIntegrationTest {
|
||||
end = System.currentTimeMillis();
|
||||
System.out.println("reanalysis analysis duration: " + (end - start));
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -840,10 +817,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -867,10 +841,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -894,10 +865,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -918,10 +886,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -943,10 +908,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -957,8 +919,10 @@ public class RedactionIntegrationTest {
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testRotationsSimple() throws IOException {
|
||||
|
||||
loadOnlyDictionaryForSimpleFile();
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
@ -970,10 +934,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/AnnotatedSimple.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -995,10 +956,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -1014,7 +972,6 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
|
||||
|
||||
@ -1034,7 +991,6 @@ public class RedactionIntegrationTest {
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
|
||||
manualRedactions.getResizeRedactions()
|
||||
.add(ManualResizeRedaction.builder()
|
||||
.annotationId("ca2b437e2480a4b5966cb8386020d454")
|
||||
@ -1084,16 +1040,8 @@ public class RedactionIntegrationTest {
|
||||
|
||||
String manualAddId = UUID.randomUUID().toString();
|
||||
|
||||
Comment comment = Comment.builder()
|
||||
.date(OffsetDateTime.now())
|
||||
.user("TEST_USER")
|
||||
.text("This is a comment test")
|
||||
.build();
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
|
||||
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
|
||||
.fileId("fileId")
|
||||
.status(AnnotationStatus.DECLINED)
|
||||
.build()));
|
||||
Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
|
||||
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
|
||||
.annotationId("675eba69b0c2917de55462c817adaa05")
|
||||
.fileId("fileId")
|
||||
@ -1112,19 +1060,8 @@ public class RedactionIntegrationTest {
|
||||
manualRedactionEntry.setType("name");
|
||||
manualRedactionEntry.setValue("O'Loughlin C.K.");
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
|
||||
.topLeftX(375.61096f)
|
||||
.topLeftY(241.282f)
|
||||
.width(7.648041f)
|
||||
.height(43.72262f)
|
||||
.page(1)
|
||||
.build(), Rectangle.builder()
|
||||
.topLeftX(384.83517f)
|
||||
.topLeftY(241.282f)
|
||||
.width(7.648041f)
|
||||
.height(17.043358f)
|
||||
.page(1)
|
||||
.build()));
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
|
||||
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
@ -1132,11 +1069,7 @@ public class RedactionIntegrationTest {
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
|
||||
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
|
||||
.fileId("fileId")
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.build()));
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.APPROVED).build()));
|
||||
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
|
||||
.annotationId("675eba69b0c2917de55462c817adaa05")
|
||||
.fileId("fileId")
|
||||
@ -1148,10 +1081,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -1183,6 +1113,7 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void classificationTestWithCvTableService() throws IOException {
|
||||
|
||||
@ -1295,10 +1226,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -1320,11 +1248,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
String manualAddId = UUID.randomUUID().toString();
|
||||
|
||||
Comment comment = Comment.builder()
|
||||
.date(OffsetDateTime.now())
|
||||
.user("TEST_USER")
|
||||
.text("This is a comment test")
|
||||
.build();
|
||||
Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
|
||||
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
|
||||
.fileId("fileId")
|
||||
@ -1354,19 +1278,8 @@ public class RedactionIntegrationTest {
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
|
||||
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
|
||||
.topLeftX(375.61096f)
|
||||
.topLeftY(241.282f)
|
||||
.width(7.648041f)
|
||||
.height(43.72262f)
|
||||
.page(1)
|
||||
.build(), Rectangle.builder()
|
||||
.topLeftX(384.83517f)
|
||||
.topLeftY(241.282f)
|
||||
.width(7.648041f)
|
||||
.height(17.043358f)
|
||||
.page(1)
|
||||
.build()));
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
|
||||
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
|
||||
|
||||
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
|
||||
@ -1397,31 +1310,10 @@ public class RedactionIntegrationTest {
|
||||
.processedDate(OffsetDateTime.now())
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
|
||||
.positions(List.of(Rectangle.builder()
|
||||
.topLeftX(289.44595f)
|
||||
.topLeftY(327.567f)
|
||||
.width(7.648041f)
|
||||
.height(82.51475f)
|
||||
.page(1)
|
||||
.build(), Rectangle.builder()
|
||||
.topLeftX(298.67056f)
|
||||
.topLeftY(327.567f)
|
||||
.width(7.648041f)
|
||||
.height(75.32377f)
|
||||
.page(1)
|
||||
.build(), Rectangle.builder()
|
||||
.topLeftX(307.89517f)
|
||||
.topLeftY(327.567f)
|
||||
.width(7.648041f)
|
||||
.height(61.670967f)
|
||||
.page(1)
|
||||
.build(), Rectangle.builder()
|
||||
.topLeftX(316.99985f)
|
||||
.topLeftY(327.567f)
|
||||
.width(7.648041f)
|
||||
.height(38.104286f)
|
||||
.page(1)
|
||||
.build()))
|
||||
.positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
|
||||
Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
|
||||
Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
|
||||
Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.build()));
|
||||
|
||||
@ -1451,10 +1343,7 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
@ -1481,13 +1370,7 @@ public class RedactionIntegrationTest {
|
||||
manualRedactionEntry.setType("CBI_author");
|
||||
manualRedactionEntry.setValue("rabbits");
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
|
||||
.topLeftX(70.944f)
|
||||
.topLeftY(670.1595f)
|
||||
.width(30.07296f)
|
||||
.height(10.048125f)
|
||||
.page(1)
|
||||
.build()));
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
|
||||
manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
|
||||
@ -1496,13 +1379,7 @@ public class RedactionIntegrationTest {
|
||||
manualRedactionEntry2.setType("CBI_author");
|
||||
manualRedactionEntry2.setValue("rabbits");
|
||||
manualRedactionEntry2.setReason("Manual Redaction");
|
||||
manualRedactionEntry2.setPositions(List.of(Rectangle.builder()
|
||||
.topLeftX(470.5204f)
|
||||
.topLeftY(746.1195f)
|
||||
.width(29.96256f)
|
||||
.height(10.048125f)
|
||||
.page(1)
|
||||
.build()));
|
||||
manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
|
||||
manualRedactionEntry3.setAnnotationId(UUID.randomUUID().toString());
|
||||
@ -1511,13 +1388,7 @@ public class RedactionIntegrationTest {
|
||||
manualRedactionEntry3.setType("CBI_author");
|
||||
manualRedactionEntry3.setValue("AOEL");
|
||||
manualRedactionEntry3.setReason("Manual Redaction");
|
||||
manualRedactionEntry3.setPositions(List.of(Rectangle.builder()
|
||||
.topLeftX(355.53775f)
|
||||
.topLeftY(266.1895f)
|
||||
.width(29.32224f)
|
||||
.height(10.048125f)
|
||||
.page(1)
|
||||
.build()));
|
||||
manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||
@ -1539,8 +1410,7 @@ public class RedactionIntegrationTest {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions)
|
||||
.getManualRedactions();
|
||||
var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions).getManualRedactions();
|
||||
surroundingTextResult.getEntriesToAdd().forEach(addEntry -> {
|
||||
assertThat(addEntry.getTextAfter()).isNotEmpty();
|
||||
});
|
||||
@ -1551,112 +1421,54 @@ public class RedactionIntegrationTest {
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
|
||||
|
||||
falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void loadOnlyDictionaryForSimpleFile() {
|
||||
|
||||
dictionary.clear();
|
||||
dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
.addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
|
||||
@ -1851,6 +1663,7 @@ public class RedactionIntegrationTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
|
||||
@ -1858,16 +1671,14 @@ public class RedactionIntegrationTest {
|
||||
|
||||
|
||||
private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
|
||||
if (entries == null)
|
||||
|
||||
if (entries == null) {
|
||||
entries = Collections.emptyList();
|
||||
}
|
||||
|
||||
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
|
||||
entries.forEach(entry -> {
|
||||
dictionaryEntries.add(DictionaryEntry.builder()
|
||||
.value(entry)
|
||||
.version(reanlysisVersions.getOrDefault(entry, 0L))
|
||||
.deleted(deleted.contains(entry))
|
||||
.build());
|
||||
dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
|
||||
});
|
||||
return dictionaryEntries;
|
||||
}
|
||||
@ -1880,17 +1691,15 @@ public class RedactionIntegrationTest {
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
|
||||
importedRedactions.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
redactionLog.getRedactionLogEntry().forEach(entry -> {
|
||||
if (entry.getValue() == null) {
|
||||
@ -1928,20 +1737,14 @@ public class RedactionIntegrationTest {
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var values = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.map(RedactionLogEntry::getValue)
|
||||
.collect(Collectors.toList());
|
||||
var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
|
||||
|
||||
assertThat(values).contains("Mrs. Robinson");
|
||||
assertThat(values).contains("Mr. Bojangles");
|
||||
@ -1973,6 +1776,7 @@ public class RedactionIntegrationTest {
|
||||
return prepareStorage(file, "files/cv_service_empty_response.json");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user