cells) {
@@ -297,7 +297,7 @@ public class Table extends AbstractPageBlock {
}
if (column != null && column.getTextBlocks() != null) {
boolean first = true;
- for (TextBlock textBlock : column.getTextBlocks()) {
+ for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
if (!first) {
sb.append("\n");
}
@@ -329,7 +329,7 @@ public class Table extends AbstractPageBlock {
sb.append(i == 0 ? "\n| " : "\n | ");
if (column != null && column.getTextBlocks() != null) {
boolean first = true;
- for (TextBlock textBlock : column.getTextBlocks()) {
+ for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
if (!first) {
sb.append("
");
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java
similarity index 91%
rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextBlock.java
rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java
index 4a6ff3fb..531a9429 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextBlock.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java
@@ -16,7 +16,7 @@ import lombok.NoArgsConstructor;
@Builder
@Data
@NoArgsConstructor
-public class TextBlock extends AbstractPageBlock {
+public class ClassificationTextBlock extends AbstractPageBlock {
@Builder.Default
private List sequences = new ArrayList<>();
@@ -175,7 +175,7 @@ public class TextBlock extends AbstractPageBlock {
}
- public TextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation, int indexOnPage) {
+ public ClassificationTextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation, int indexOnPage) {
this.indexOnPage = indexOnPage;
this.minX = minX;
@@ -187,23 +187,23 @@ public class TextBlock extends AbstractPageBlock {
}
- public TextBlock union(TextPositionSequence r) {
+ public ClassificationTextBlock union(TextPositionSequence r) {
- TextBlock union = this.copy();
+ ClassificationTextBlock union = this.copy();
union.add(r);
return union;
}
- public TextBlock union(TextBlock r) {
+ public ClassificationTextBlock union(ClassificationTextBlock r) {
- TextBlock union = this.copy();
+ ClassificationTextBlock union = this.copy();
union.add(r);
return union;
}
- public void add(TextBlock r) {
+ public void add(ClassificationTextBlock r) {
if (r.getMinX() < minX) {
minX = r.getMinX();
@@ -238,9 +238,9 @@ public class TextBlock extends AbstractPageBlock {
}
- public TextBlock copy() {
+ public ClassificationTextBlock copy() {
- return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
+ return new ClassificationTextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java
index fafc436e..8522892c 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java
@@ -35,7 +35,7 @@ public class SectionText {
@Builder.Default
private Set images = new HashSet<>();
@Builder.Default
- private List textBlocks = new ArrayList<>();
+ private List textBlocks = new ArrayList<>();
@Builder.Default
private Map tabularData = new HashMap<>();
@Builder.Default
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java
index 36fb8bcb..6ce3de1e 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java
@@ -12,7 +12,7 @@ import lombok.Data;
@AllArgsConstructor
public class UnclassifiedText {
- private List textBlocks;
+ private List textBlocks;
@JsonIgnore
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java
index 75328630..84b15272 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java
@@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil;
@@ -29,7 +29,7 @@ public class BlockificationService {
/**
* This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
* This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
- * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
+ * Rulings (TablePageBlock lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
*
* @param textPositions The words of a page.
* @param horizontalRulingLines Horizontal table lines.
@@ -64,7 +64,7 @@ public class BlockificationService {
prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
}
- TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+ ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
indexOnPage++;
chunkBlockList1.add(cb1);
@@ -106,17 +106,17 @@ public class BlockificationService {
}
}
- TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+ ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
if (cb1 != null) {
chunkBlockList1.add(cb1);
}
Iterator itty = chunkBlockList1.iterator();
- TextBlock previousLeft = null;
- TextBlock previousRight = null;
+ ClassificationTextBlock previousLeft = null;
+ ClassificationTextBlock previousRight = null;
while (itty.hasNext()) {
- TextBlock block = (TextBlock) itty.next();
+ ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
@@ -142,9 +142,9 @@ public class BlockificationService {
}
itty = chunkBlockList1.iterator();
- TextBlock previous = null;
+ ClassificationTextBlock previous = null;
while (itty.hasNext()) {
- TextBlock block = (TextBlock) itty.next();
+ ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
@@ -167,9 +167,9 @@ public class BlockificationService {
}
- private TextBlock buildTextBlock(List wordBlockList, int indexOnPage) {
+ private ClassificationTextBlock buildTextBlock(List wordBlockList, int indexOnPage) {
- TextBlock textBlock = null;
+ ClassificationTextBlock textBlock = null;
FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
@@ -186,7 +186,7 @@ public class BlockificationService {
styleFrequencyCounter.add(wordBlock.getFontStyle());
if (textBlock == null) {
- textBlock = new TextBlock(wordBlock.getMinXDirAdj(),
+ textBlock = new ClassificationTextBlock(wordBlock.getMinXDirAdj(),
wordBlock.getMaxXDirAdj(),
wordBlock.getMinYDirAdj(),
wordBlock.getMaxYDirAdj(),
@@ -194,7 +194,7 @@ public class BlockificationService {
wordBlock.getRotation(),
indexOnPage);
} else {
- TextBlock spatialEntity = textBlock.union(wordBlock);
+ ClassificationTextBlock spatialEntity = textBlock.union(wordBlock);
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java
index 30bd9ac5..d4657f5e 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java
@@ -10,8 +10,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
@Service
@@ -77,8 +77,8 @@ public class BodyTextFrameService {
for (AbstractPageBlock container : page.getTextBlocks()) {
- if (container instanceof TextBlock) {
- TextBlock textBlock = (TextBlock) container;
+ if (container instanceof ClassificationTextBlock) {
+ ClassificationTextBlock textBlock = (ClassificationTextBlock) container;
if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
continue;
}
@@ -94,15 +94,15 @@ public class BodyTextFrameService {
}
}
- if (container instanceof Table) {
- Table table = (Table) container;
+ if (container instanceof TablePageBlock) {
+ TablePageBlock table = (TablePageBlock) container;
for (List row : table.getRows()) {
for (Cell cell : row) {
if (cell == null || cell.getTextBlocks() == null) {
continue;
}
- for (TextBlock textBlock : cell.getTextBlocks()) {
+ for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
expandRectangle(textBlock, page, expansionsRectangle);
}
}
@@ -117,7 +117,7 @@ public class BodyTextFrameService {
}
- private void expandRectangle(TextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
+ private void expandRectangle(ClassificationTextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
if (textBlock.getPdfMinY() < expansionsRectangle.minX) {
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java
index 8764ec39..8ab701f7 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java
@@ -9,7 +9,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
import lombok.RequiredArgsConstructor;
@@ -41,14 +41,14 @@ public class ClassificationService {
public void classifyPage(Page page, Document document, List headlineFontSizes) {
for (AbstractPageBlock textBlock : page.getTextBlocks()) {
- if (textBlock instanceof TextBlock) {
- classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
+ if (textBlock instanceof ClassificationTextBlock) {
+ classifyBlock((ClassificationTextBlock) textBlock, page, document, headlineFontSizes);
}
}
}
- public void classifyBlock(TextBlock textBlock, Page page, Document document, List headlineFontSizes) {
+ public void classifyBlock(ClassificationTextBlock textBlock, Page page, Document document, List headlineFontSizes) {
var bodyTextFrame = page.getBodyTextFrame();
@@ -84,7 +84,7 @@ public class ClassificationService {
document.setHeadlines(true);
}
}
- } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+ } else if (!textBlock.getText().startsWith("TablePageBlock ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
.getMostPopular()
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java
index 2375ce27..5fffa35f 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java
@@ -23,7 +23,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
@@ -154,11 +154,11 @@ public class PdfSegmentationService {
// Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
for (AbstractPageBlock textBlock : page.getTextBlocks()) {
- if (textBlock instanceof TextBlock) {
- if (((TextBlock) textBlock).getSequences() == null) {
+ if (textBlock instanceof ClassificationTextBlock) {
+ if (((ClassificationTextBlock) textBlock).getSequences() == null) {
continue;
}
- for (TextPositionSequence word : ((TextBlock) textBlock).getSequences()) {
+ for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) {
page.getTextHeightCounter().add(word.getTextHeight());
page.getFontCounter().add(word.getFont());
page.getFontSizeCounter().add(word.getFontSize());
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java
index d3058889..468b7793 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java
@@ -18,8 +18,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
import lombok.extern.slf4j.Slf4j;
@@ -39,11 +39,11 @@ public class SectionsBuilderService {
AbstractPageBlock prev = null;
String lastHeadline = "";
- Table previousTable = null;
+ TablePageBlock previousTable = null;
for (Page page : document.getPages()) {
- List header = new ArrayList<>();
- List footer = new ArrayList<>();
- List unclassifiedText = new ArrayList<>();
+ List header = new ArrayList<>();
+ List footer = new ArrayList<>();
+ List unclassifiedText = new ArrayList<>();
for (AbstractPageBlock current : page.getTextBlocks()) {
if (current.getClassification() == null) {
@@ -53,17 +53,17 @@ public class SectionsBuilderService {
current.setPage(page.getPageNumber());
if (current.getClassification().equals("Header")) {
- header.add((TextBlock) current);
+ header.add((ClassificationTextBlock) current);
continue;
}
if (current.getClassification().equals("Footer")) {
- footer.add((TextBlock) current);
+ footer.add((ClassificationTextBlock) current);
continue;
}
if (current.getClassification().equals("Other")) {
- unclassifiedText.add((TextBlock) current);
+ unclassifiedText.add((ClassificationTextBlock) current);
continue;
}
@@ -79,7 +79,7 @@ public class SectionsBuilderService {
previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
}
}
- if (current instanceof Table table) {
+ if (current instanceof TablePageBlock table) {
// Distribute header information for subsequent tables
mergeTableMetadata(table, previousTable);
previousTable = table;
@@ -212,7 +212,7 @@ public class SectionsBuilderService {
}
- private void mergeTableMetadata(Table currentTable, Table previousTable) {
+ private void mergeTableMetadata(TablePageBlock currentTable, TablePageBlock previousTable) {
// Distribute header information for subsequent tables
if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
@@ -245,39 +245,39 @@ public class SectionsBuilderService {
Section section = new Section();
for (AbstractPageBlock container : wordBlockList) {
- if (container instanceof Table table) {
+ if (container instanceof TablePageBlock table) {
if (lastHeadline == null || lastHeadline.isEmpty()) {
table.setHeadline("Text in table");
} else {
- table.setHeadline("Table in: " + lastHeadline);
+ table.setHeadline("TablePageBlock in: " + lastHeadline);
}
section.getPageBlocks().add(table);
continue;
}
- TextBlock wordBlock = (TextBlock) container;
+ ClassificationTextBlock wordBlock = (ClassificationTextBlock) container;
section.getPageBlocks().add(wordBlock);
}
return section;
}
- private boolean hasValidHeaderInformation(Table table) {
+ private boolean hasValidHeaderInformation(TablePageBlock table) {
return !hasInvalidHeaderInformation(table);
}
- private boolean hasInvalidHeaderInformation(Table table) {
+ private boolean hasInvalidHeaderInformation(TablePageBlock table) {
return table.getRows().stream().flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells()))).findAny().isEmpty();
}
- private List getRowWithNonHeaderCells(Table table) {
+ private List getRowWithNonHeaderCells(TablePageBlock table) {
for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
List row = table.getRows().get(i);
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java
index ff2f6d6e..fede5d3c 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java
@@ -19,8 +19,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
@Service
@@ -83,10 +83,10 @@ public class TableExtractionService {
List cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
- List toBeRemoved = new ArrayList<>();
+ List toBeRemoved = new ArrayList<>();
for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
- TextBlock textBlock = (TextBlock) abstractPageBlock;
+ ClassificationTextBlock textBlock = (ClassificationTextBlock) abstractPageBlock;
for (Cell cell : cells) {
if (cell.intersects(textBlock.getPdfMinX(),
textBlock.getPdfMinY(),
@@ -104,7 +104,7 @@ public class TableExtractionService {
List spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).collect(Collectors.toList());
- List tables = new ArrayList<>();
+ List tables = new ArrayList<>();
for (Rectangle area : spreadsheetAreas) {
List overlappingCells = new ArrayList<>();
@@ -113,16 +113,16 @@ public class TableExtractionService {
overlappingCells.add(c);
}
}
- tables.add(new Table(overlappingCells, area, page.getRotation()));
+ tables.add(new TablePageBlock(overlappingCells, area, page.getRotation()));
}
- for (Table table : tables) {
+ for (TablePageBlock table : tables) {
int position = -1;
Iterator itty = page.getTextBlocks().iterator();
while (itty.hasNext()) {
AbstractPageBlock textBlock = itty.next();
- if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) {
+ if (textBlock instanceof ClassificationTextBlock ? table.containsBlock((ClassificationTextBlock) textBlock) : table.contains(textBlock) && position == -1) {
position = page.getTextBlocks().indexOf(textBlock);
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java
index f7d1d48e..63ccfa01 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java
@@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import lombok.experimental.UtilityClass;
@@ -11,7 +11,7 @@ public final class PositionUtils {
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
- public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
+ public boolean isWithinBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
if (btf == null || textBlock == null) {
return false;
@@ -32,7 +32,7 @@ public final class PositionUtils {
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
- public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
+ public boolean isOverBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
if (btf == null || textBlock == null) {
return false;
@@ -61,7 +61,7 @@ public final class PositionUtils {
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
- public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
+ public boolean isUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
if (btf == null || textBlock == null) {
return false;
@@ -90,7 +90,7 @@ public final class PositionUtils {
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
- public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
+ public boolean isTouchingUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
//TODO Currently this is not working for rotated pages.
@@ -107,13 +107,13 @@ public final class PositionUtils {
}
- public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
+ public float getHeightDifferenceBetweenChunkWordAndDocumentWord(ClassificationTextBlock textBlock, Float documentMostPopularWordHeight) {
return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
}
- public Float getApproxLineCount(TextBlock textBlock) {
+ public Float getApproxLineCount(ClassificationTextBlock textBlock) {
return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java
index 49873945..d5b3c1a5 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java
@@ -10,10 +10,11 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Ato
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
@@ -69,6 +70,7 @@ public class DocumentDataMapper {
case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
+ case SECTION -> PropertiesMapper.buildSectionProperties((SectionNode) entry.getNode());
default -> new HashMap<>();
};
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
index db7343fd..5e3b64c3 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
@@ -18,8 +18,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Doc
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
@@ -71,7 +71,7 @@ public class DocumentGraphMapper {
List pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
SemanticNode node = switch (entryData.getType()) {
- case SECTION -> buildSection(context);
+ case SECTION -> buildSection(context, entryData.getProperties());
case PARAGRAPH -> buildParagraph(context, terminal);
case HEADLINE -> buildHeadline(context, terminal);
case HEADER -> buildHeader(context, terminal);
@@ -134,7 +134,7 @@ public class DocumentGraphMapper {
TableNode.TableNodeBuilder builder = TableNode.builder();
PropertiesMapper.parseTableProperties(properties, builder);
- return TableNode.builder().tableOfContents(context.tableOfContents()).build();
+ return builder.tableOfContents(context.tableOfContents()).build();
}
@@ -150,10 +150,11 @@ public class DocumentGraphMapper {
}
- private SectionNode buildSection(Context context) {
-
- return SectionNode.builder().tableOfContents(context.tableOfContents()).build();
+ private SectionNode buildSection(Context context, Map properties) {
+ var builder = SectionNode.builder();
+ PropertiesMapper.parseSectionProperties(properties, builder);
+ return builder.tableOfContents(context.tableOfContents()).build();
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java
index 6388d8f7..8468a719 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java
@@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
@@ -50,6 +51,14 @@ public class PropertiesMapper {
}
+ public static Map buildSectionProperties(SectionNode node) {
+
+ Map properties = new HashMap<>();
+ properties.put("excludesTables", String.valueOf(node.isExcludesTables()));
+ return properties;
+ }
+
+
public static void parseImageProperties(Map properties, ImageNode.ImageNodeBuilder builder) {
builder.imageType(parseImageType(properties.get("imageType")));
@@ -75,6 +84,12 @@ public class PropertiesMapper {
}
+ public static void parseSectionProperties(Map properties, SectionNode.SectionNodeBuilder builder) {
+
+ builder.excludesTables(Boolean.parseBoolean(properties.get("excludesTables")));
+ }
+
+
public static ImageType parseImageType(String imageType) {
return switch (imageType) {
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java
index 04c97823..2c78a62a 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java
@@ -1,13 +1,12 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import static java.lang.String.format;
+import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import java.awt.geom.Rectangle2D;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -15,8 +14,6 @@ import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
-import org.springframework.stereotype.Service;
-
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
@@ -24,11 +21,11 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
@@ -45,7 +42,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-@Service
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
public class DocumentGraphFactory {
public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
@@ -57,7 +56,7 @@ public class DocumentGraphFactory {
DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
- document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
+ document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
addSections(document, context);
addHeaderAndFooterToEachPage(document, context);
@@ -72,68 +71,11 @@ public class DocumentGraphFactory {
private void addSections(Document document, Context context) {
- document.getSections().forEach(section -> addSection(null, section.getPageBlocks(), section.getImages(), context));
+ document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getPageBlocks(), section.getImages(), context));
}
- private void addSection(SemanticNode parentNode, List pageBlocks, List images, Context context) {
-
- Map> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
- SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
-
- context.sections().add(sectionNode);
- blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
-
- List tocId;
- if (parentNode == null) {
- tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
- } else {
- tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
- }
- sectionNode.setTocId(tocId);
- Set alreadyMerged = new HashSet<>();
- for (AbstractPageBlock abstractPageBlock : pageBlocks) {
-
- if (alreadyMerged.contains(abstractPageBlock)) {
- continue;
- }
-
- if (abstractPageBlock instanceof TextBlock) {
- List textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
- alreadyMerged.addAll(textBlocks);
- addParagraphOrHeadline(sectionNode, (TextBlock) abstractPageBlock, context, textBlocks);
- }
- if (abstractPageBlock instanceof Table) {
- addTable(sectionNode, (Table) abstractPageBlock, context);
- }
- }
- for (ClassifiedImage image : images) {
-
- addImage(sectionNode, image, context);
- }
- }
-
-
- private static List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) {
-
- return pageBlocks.stream()
- .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
- .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
- .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock)
- .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
- .map(abstractTextContainer -> (TextBlock) abstractTextContainer)
- .toList();
- }
-
-
- private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) {
-
- PageNode page = getPage(pageNumber, context);
- page.getMainBody().add(sectionNode);
- }
-
-
- private void addTable(SemanticNode parentNode, Table table, Context context) {
+ public void addTable(SemanticNode parentNode, TablePageBlock table, Context context) {
PageNode page = getPage(table.getPage(), context);
TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
@@ -151,11 +93,11 @@ public class DocumentGraphFactory {
}
- private void addTableCells(List> rows, SemanticNode parentNode, Context context, int pageNumber) {
+ private void addTableCells(List> rows, TableNode tableNode, Context context, int pageNumber) {
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
- addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context);
+ addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, pageNumber, context);
}
}
}
@@ -190,7 +132,7 @@ public class DocumentGraphFactory {
tableCellNode.setTerminal(true);
} else if (firstTextBlockIsHeadline(cell)) {
- addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), Collections.emptyList(), context);
+ SectionNodeFactory.addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
@@ -220,18 +162,24 @@ public class DocumentGraphFactory {
}
- private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context) {
+ public static boolean pageBlockIsHeadline(AbstractPageBlock abstractPageBlock) {
- addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
+ return abstractPageBlock instanceof ClassificationTextBlock && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().startsWith("H");
}
- private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context, List textBlocksToMerge) {
+ private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context) {
+
+ addParagraphOrHeadline(parentNode, originalTextBlock, context, emptyList());
+ }
+
+
+ public void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, List textBlocksToMerge) {
PageNode page = getPage(originalTextBlock.getPage(), context);
SemanticNode node;
- if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
+ if (pageBlockIsHeadline(originalTextBlock)) {
node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
} else {
node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -239,7 +187,7 @@ public class DocumentGraphFactory {
page.getMainBody().add(node);
- List textBlocks = new LinkedList<>(textBlocksToMerge);
+ List textBlocks = new LinkedList<>(textBlocksToMerge);
textBlocks.add(originalTextBlock);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
@@ -256,7 +204,7 @@ public class DocumentGraphFactory {
}
- private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
+ public void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
PageNode page = getPage(image.getPage(), context);
@@ -277,13 +225,13 @@ public class DocumentGraphFactory {
private void addHeaderAndFooterToEachPage(Document document, Context context) {
- Map> headers = document.getHeaders()
+ Map> headers = document.getHeaders()
.stream()
.map(Header::getTextBlocks)
.flatMap(List::stream)
.collect(groupingBy(AbstractPageBlock::getPage, toList()));
- Map> footers = document.getFooters()
+ Map> footers = document.getFooters()
.stream()
.map(Footer::getTextBlocks)
.flatMap(List::stream)
@@ -307,7 +255,7 @@ public class DocumentGraphFactory {
}
- private void addFooter(List textBlocks, Context context) {
+ private void addFooter(List textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -322,7 +270,7 @@ public class DocumentGraphFactory {
}
- public void addHeader(List textBlocks, Context context) {
+ public void addHeader(List textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -374,7 +322,7 @@ public class DocumentGraphFactory {
}
- private PageNode getPage(int pageIndex, Context context) {
+ public PageNode getPage(int pageIndex, Context context) {
return context.pages.keySet()
.stream()
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
new file mode 100644
index 00000000..68cbefe4
--- /dev/null
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
@@ -0,0 +1,187 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
+
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory.pageBlockIsHeadline;
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SectionNodeFactory {
+
+ public void addSection(SemanticNode parentNode, List pageBlocks, List images, DocumentGraphFactory.Context context) {
+
+ if (pageBlocks.isEmpty()) {
+ return;
+ }
+ Map> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
+ SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
+
+ context.sections().add(sectionNode);
+ blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
+
+ List tocId;
+ if (parentNode == null) {
+ tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
+ } else {
+ tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
+ }
+ sectionNode.setTocId(tocId);
+
+ addFirstHeadlineDirectlyToSection(pageBlocks, context, sectionNode);
+ if (containsTablesAndTextBlocks(pageBlocks)) {
+ splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(sectionNode, subSectionPageBlocks, emptyList(), context));
+ } else {
+ addTablesOrParagraphsOrHeadlinesToSection(pageBlocks, context, sectionNode);
+ if (listIsTextBlockOnly(pageBlocks)) {
+ sectionNode.setExcludesTables(true);
+ }
+ }
+
+ for (ClassifiedImage image : images) {
+ DocumentGraphFactory.addImage(sectionNode, image, context);
+ }
+ }
+
+
+ private static boolean listIsTextBlockOnly(List pageBlocks) {
+
+ return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock);
+ }
+
+
+ private void addFirstHeadlineDirectlyToSection(List pageBlocks, DocumentGraphFactory.Context context, SectionNode sectionNode) {
+
+ if (pageBlockIsHeadline(pageBlocks.get(0))) {
+ addTablesOrParagraphsOrHeadlinesToSection(List.of(pageBlocks.get(0)), context, sectionNode);
+ pageBlocks.remove(0);
+ }
+ }
+
+
+ private void addTablesOrParagraphsOrHeadlinesToSection(List pageBlocks, DocumentGraphFactory.Context context, SectionNode sectionNode) {
+
+ Set alreadyMerged = new HashSet<>();
+ for (AbstractPageBlock abstractPageBlock : pageBlocks) {
+
+ if (alreadyMerged.contains(abstractPageBlock)) {
+ continue;
+ }
+
+ if (abstractPageBlock instanceof ClassificationTextBlock) {
+ List textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
+ alreadyMerged.addAll(textBlocks);
+ DocumentGraphFactory.addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractPageBlock, context, textBlocks);
+ }
+ if (abstractPageBlock instanceof TablePageBlock) {
+ DocumentGraphFactory.addTable(sectionNode, (TablePageBlock) abstractPageBlock, context);
+ }
+ }
+ }
+
+
+ private static boolean containsTablesAndTextBlocks(List pageBlocks) {
+
+ return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+ .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock);
+ }
+
+
+ /**
+ * This function splits the list of PageBlocks around TablePageBlocks, such that SubSections can be created, that don't include tables.
+ * First, the function splits the list into coherent Lists, which include only either type.
+ * If a Table is directly preceded by a Headline, it is moved to the Table list.
+ *
+ * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
+ * @return List of Lists of AbstractPageBlocks, which include either a single Headline ClassificationTextBlock and a TablePageBlock or only ClassificationTextBlocks.
+ */
+ private List> splitPageBlocksIntoSubSections(List pageBlocks) {
+
+ List> splitList = splitIntoCoherentList(pageBlocks);
+ movePrecedingHeadlineToTableList(splitList);
+ return splitList.stream().filter(list -> !list.isEmpty()).toList();
+ }
+
+
+ private static void movePrecedingHeadlineToTableList(List> splitList) {
+
+ for (int i = 0; i < splitList.size(); i++) {
+ if (listIsTableOnly(splitList.get(i))) {
+ if (i > 0) {
+ List previousList = splitList.get(i - 1);
+ AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1);
+ if (pageBlockIsHeadline(lastPageBlockInPreviousList)) {
+ previousList.remove(i - 1);
+ splitList.get(i).add(0, lastPageBlockInPreviousList);
+ }
+ }
+ }
+ }
+ }
+
+
+ private static boolean listIsTableOnly(List abstractPageBlocks) {
+
+ return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
+ }
+
+
+ /**
+ * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
+ * @return List of Lists of AbstractPageBlocks, which are exclusively of type ClassificationTextBlock or TablePageBlock
+ */
+ private static List> splitIntoCoherentList(List pageBlocks) {
+
+ List> splitList = new LinkedList<>();
+ List currentList = new LinkedList<>();
+ splitList.add(currentList);
+
+ Class lastPageBlockClass = pageBlocks.get(0).getClass();
+ for (AbstractPageBlock pageBlock : pageBlocks) {
+ if (lastPageBlockClass.isInstance(pageBlock)) {
+ currentList.add(pageBlock);
+ } else {
+ currentList = new LinkedList<>();
+ currentList.add(pageBlock);
+ splitList.add(currentList);
+ lastPageBlockClass = pageBlock.getClass();
+ }
+ }
+ return splitList;
+ }
+
+
+ private static List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) {
+
+ return pageBlocks.stream()
+ .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
+ .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
+ .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock)
+ .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
+ .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer)
+ .toList();
+ }
+
+
+ private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, SectionNode sectionNode, Integer pageNumber) {
+
+ PageNode page = DocumentGraphFactory.getPage(pageNumber, context);
+ page.getMainBody().add(sectionNode);
+ }
+
+}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java
index 3def39e7..467bfe9f 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java
@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
import static java.lang.String.format;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -105,7 +106,7 @@ public class Boundary implements Comparable {
}
- public static Boundary merge(List boundaries) {
+ public static Boundary merge(Collection boundaries) {
int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java
index 0329bf79..f51276a5 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java
@@ -9,6 +9,7 @@ import java.util.List;
import java.util.stream.Stream;
import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
@@ -84,7 +85,7 @@ public class TableOfContents {
public boolean hasParentById(List tocId) {
- return entryExists(getParentId(tocId));
+ return !tocId.isEmpty();
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
index 9194dcb7..a4fbe035 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
@@ -45,14 +45,15 @@ public class RedactionEntity {
boolean dossierDictionaryEntry;
Set engines;
Set references;
- int matchedRule;
+ @Builder.Default
+ List matchedRules = new LinkedList<>();
String redactionReason;
String legalBasis;
// inferred on graph insertion
String value;
- CharSequence textBefore;
- CharSequence textAfter;
+ String textBefore;
+ String textAfter;
@Builder.Default
Set pages = new HashSet<>();
List redactionPositionsPerPage;
@@ -76,13 +77,30 @@ public class RedactionEntity {
.dossierDictionaryEntry(false)
.engines(new HashSet<>())
.references(new HashSet<>())
- .matchedRule(-1)
.redactionReason("")
.legalBasis("")
.build();
}
+ public boolean occursInNodeOfType(Class clazz) {
+
+ return intersectingNodes.stream().anyMatch(clazz::isInstance);
+ }
+
+
+ public boolean isType(String type) {
+
+ return this.type.equals(type);
+ }
+
+
+ public boolean isAnyType(List types) {
+
+ return types.contains(type);
+ }
+
+
public void addIntersectingNode(SemanticNode containingNode) {
intersectingNodes.add(containingNode);
@@ -91,10 +109,29 @@ public class RedactionEntity {
public void removeFromGraph() {
- getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
+ intersectingNodes.forEach(node -> node.getEntities().remove(this));
+ pages.forEach(page -> page.getEntities().remove(this));
+
+ intersectingNodes = new LinkedList<>();
deepestFullyContainingNode = null;
- getPages().forEach(page -> page.getEntities().remove(this));
- setRemoved(true);
+ pages = new HashSet<>();
+ removed = true;
+ }
+
+
+ public void addMatchedRule(int ruleNumber) {
+
+ matchedRules.add(ruleNumber);
+ engines.add(Engine.RULE);
+ }
+
+
+ public int getMatchedRule() {
+
+ if (matchedRules.isEmpty()) {
+ return -1;
+ }
+ return matchedRules.get(matchedRules.size() - 1);
}
@@ -155,9 +192,15 @@ public class RedactionEntity {
}
- public void addReference(RedactionEntity redactionEntity) {
+ public void addReference(RedactionEntity reference) {
- references.add(redactionEntity);
+ references.add(reference);
+ }
+
+
+ public void addReferences(List references) {
+
+ this.references.addAll(references);
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/DocumentGraph.java
similarity index 80%
rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentGraph.java
rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/DocumentGraph.java
index e7a52cbc..f7285049 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentGraph.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/DocumentGraph.java
@@ -1,19 +1,18 @@
-package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
@@ -35,6 +34,8 @@ public class DocumentGraph implements SemanticNode {
TableOfContents tableOfContents;
Integer numberOfPages;
TextBlock textBlock;
+ @Builder.Default
+ Set entities = new HashSet<>();
public TextBlock buildTextBlock() {
@@ -55,12 +56,6 @@ public class DocumentGraph implements SemanticNode {
}
- public Set getEntities() {
-
- return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
- }
-
-
@Override
public List getTocId() {
@@ -71,7 +66,17 @@ public class DocumentGraph implements SemanticNode {
@Override
public void setTocId(List tocId) {
- throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
+ throw new UnsupportedOperationException("DocumentGraph is always the root of the TablePageBlock of Contents");
+ }
+
+
+ @Override
+ public HeadlineNode getHeadline() {
+
+ return streamAllSubNodes().filter(node -> node instanceof HeadlineNode)
+ .map(node -> (HeadlineNode) node)
+ .findFirst()
+ .orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java
index a6af7772..b2cb28a9 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java
@@ -52,7 +52,7 @@ public class HeadlineNode implements SemanticNode {
@Override
- public SemanticNode getHeadline() {
+ public HeadlineNode getHeadline() {
return this;
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java
index 30f906e9..5e8a7f99 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java
@@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
@@ -30,6 +29,7 @@ public class SectionNode implements SemanticNode {
TextBlock textBlock;
@EqualsAndHashCode.Exclude
TableOfContents tableOfContents;
+ boolean excludesTables;
@Builder.Default
@EqualsAndHashCode.Exclude
@@ -55,10 +55,11 @@ public class SectionNode implements SemanticNode {
public HeadlineNode getHeadline() {
- return streamChildren().filter(node -> node instanceof HeadlineNode)
- .map(node -> (HeadlineNode) node)
- .findFirst()
- .orElseThrow(() -> new NotFoundException("Section has no Headline!"));
+ return streamChildren()//
+ .filter(node -> node instanceof HeadlineNode)//
+ .map(node -> (HeadlineNode) node)//
+ .findFirst()//
+ .orElseGet(() -> getParent().getHeadline());
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
index 650c406b..3070df44 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
@@ -91,7 +91,7 @@ public interface SemanticNode {
*
* @return First HeadlineNode found
*/
- default SemanticNode getHeadline() {
+ default HeadlineNode getHeadline() {
return getParent().getHeadline();
}
@@ -157,6 +157,24 @@ public interface SemanticNode {
}
+ default boolean hasEntitiesOfType(String type) {
+
+ return getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+ }
+
+
+ default List getEntitiesOfType(String type) {
+
+ return getEntities().stream().filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
+ }
+
+
+ default List getEntitiesOfType(List types) {
+
+ return getEntities().stream().filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
+ }
+
+
/**
* Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node.
* If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1.
@@ -184,8 +202,8 @@ public interface SemanticNode {
/**
- * @param string A String which the ClassificationTextBlock might contain
- * @return true, if this node's ClassificationTextBlock contains the string
+ * @param string A String which the TextBlock might contain
+ * @return true, if this node's TextBlock contains the string
*/
default boolean containsString(String string) {
@@ -193,6 +211,26 @@ public interface SemanticNode {
}
+ /**
+ * @param strings A List of Strings which the TextBlock might contain
+ * @return true, if this node's TextBlock contains all strings
+ */
+ default boolean containsStrings(List strings) {
+
+ return strings.stream().allMatch(this::containsString);
+ }
+
+
+ /**
+ * @param string A String which the ClassificationTextBlock might contain
+ * @return true, if this node's ClassificationTextBlock contains the string
+ */
+ default boolean containsStringIgnoreCase(String string) {
+
+ return buildTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+ }
+
+
/**
* @param strings A List of Strings which the ClassificationTextBlock might contain
* @return true, if this node's ClassificationTextBlock contains any of the strings
@@ -203,6 +241,16 @@ public interface SemanticNode {
}
+ /**
+ * @param strings A List of Strings which the ClassificationTextBlock might contain
+ * @return true, if this node's ClassificationTextBlock contains any of the strings
+ */
+ default boolean containsAnyStringIgnoreCase(List strings) {
+
+ return strings.stream().anyMatch(this::containsStringIgnoreCase);
+ }
+
+
/**
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
* It sets the fields accordingly and recursively calls this function on all its children.
@@ -213,13 +261,13 @@ public interface SemanticNode {
TextBlock textBlock = buildTextBlock();
if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
-
if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
redactionEntity.setDeepestFullyContainingNode(this);
}
redactionEntity.addIntersectingNode(this);
- streamChildren().forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
+ streamChildren().filter(semanticNode -> semanticNode.getBoundary().intersects(redactionEntity.getBoundary()))
+ .forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java
index ad3b23c7..e23d3f75 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java
@@ -1,8 +1,12 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
+import static java.lang.String.format;
+
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
@@ -26,8 +30,8 @@ public class TableNode implements SemanticNode {
List tocId;
TableOfContents tableOfContents;
- Integer numberOfRows;
- Integer numberOfCols;
+ int numberOfRows;
+ int numberOfCols;
TextBlock textBlock;
@@ -36,12 +40,95 @@ public class TableNode implements SemanticNode {
Set entities = new HashSet<>();
+ public Stream streamEntitiesWhereRowContainsStringsIgnoreCase(List strings) {
+
+ return IntStream.range(0, numberOfRows)
+ .boxed()
+ .filter(row -> rowContainsStringsIgnoreCase(row, strings))
+ .flatMap(this::streamRow)
+ .map(TableCellNode::getEntities)
+ .flatMap(Collection::stream);
+ }
+
+
+ public boolean rowContainsStringsIgnoreCase(Integer row, List strings) {
+
+ String rowText = streamRow(row).map(TableCellNode::buildTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+ return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+ }
+
+
+ public Stream streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
+
+ List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCellNode::getCol).toList();
+ return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
+ .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+ .map(TableCellNode::getEntities)
+ .flatMap(Collection::stream);
+ }
+
+
+ public Stream streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List values) {
+
+ List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCellNode::getCol).toList();
+ return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
+ .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsAnyString(values)))
+ .map(TableCellNode::getEntities)
+ .flatMap(Collection::stream);
+ }
+
+
+ public Stream streamEntitiesWhereRowContainsEntitiesOfType(String type) {
+
+ List rowsWithEntityOfType = getEntities().stream()
+ .filter(redactionEntity -> redactionEntity.getType().equals(type))
+ .map(RedactionEntity::getIntersectingNodes)
+ .filter(node -> node instanceof TableCellNode)
+ .map(node -> (TableCellNode) node)
+ .map(TableCellNode::getRow)
+ .toList();
+
+ return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCellNode::getEntities).flatMap(Collection::stream);
+ }
+
+
+ public TableCellNode getCell(int row, int col) {
+
+ if (numberOfRows - row < 0 || numberOfCols - col < 0) {
+ throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
+ }
+ int idx = row * numberOfCols + col;
+ return (TableCellNode) tableOfContents.getEntryById(tocId).getChildren().get(idx).getNode();
+ }
+
+
public Stream streamTableCells() {
return streamChildren().map(node -> (TableCellNode) node);
}
+ public Stream streamTableCellsWithHeader(String header) {
+
+ return streamHeaders().filter(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header))
+ .map(TableCellNode::getCol)
+ .flatMap(this::streamCol)
+ .filter(tableCellNode -> !tableCellNode.isHeader());
+ }
+
+
+ public Stream streamCol(int col) {
+
+ return streamTableCells().filter(tableCellNode -> tableCellNode.getCol() == col);
+ }
+
+
+ public Stream streamRow(int row) {
+
+ return streamTableCells().filter(tableCellNode -> tableCellNode.getRow() == row);
+ }
+
+
public Stream streamHeaders() {
return streamTableCells().filter(TableCellNode::isHeader);
@@ -54,6 +141,24 @@ public class TableNode implements SemanticNode {
}
+ public boolean hasHeader(String header) {
+
+ return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header));
+ }
+
+
+ public boolean hasRowWithHeaderAndValue(String header, String value) {
+
+ return streamTableCellsWithHeader(header).anyMatch(tableCellNode -> tableCellNode.containsString(value));
+ }
+
+
+ public boolean hasRowWithHeaderAndAnyValue(String header, List values) {
+
+ return streamTableCellsWithHeader(header).anyMatch(tableCellNode -> tableCellNode.containsAnyString(values));
+ }
+
+
@Override
public TextBlock buildTextBlock() {
@@ -67,7 +172,7 @@ public class TableNode implements SemanticNode {
@Override
public String toString() {
- return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
+ return tocId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + buildTextBlock().buildSummary();
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java
index e44ab2ca..fe353cc0 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java
@@ -7,6 +7,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
@@ -162,10 +163,7 @@ public class ConcatenatedTextBlock implements TextBlock {
private Map> mergeEntityPositionsWithSamePageNode(Map> map1, Map> map2) {
Map> mergedMap = new HashMap<>(map1);
- map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (rects1, rects2) -> {
- rects1.addAll(rects2);
- return rects1;
- }));
+ map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (l1, l2) -> Stream.concat(l1.stream(), l2.stream()).toList()));
return mergedMap;
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
index fdb582bd..135919da 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
@@ -1,12 +1,16 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedEndByRegex;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedStartByRegex;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
+import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -16,12 +20,15 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.CharSequenceSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -37,8 +44,8 @@ public class EntityCreationService {
public Set betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
- List startBoundaries = CharSequenceSearchUtils.findBoundariesByString(start, textBlock);
- List stopBoundaries = CharSequenceSearchUtils.findBoundariesByString(stop, textBlock);
+ List startBoundaries = RedactionSearchUtils.findBoundariesByString(start, textBlock);
+ List stopBoundaries = RedactionSearchUtils.findBoundariesByString(stop, textBlock);
List entityBoundaries = new LinkedList<>();
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
@@ -85,7 +92,7 @@ public class EntityCreationService {
public Set lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
- List boundaries = CharSequenceSearchUtils.findBoundariesByString(string, textBlock);
+ List boundaries = RedactionSearchUtils.findBoundariesByString(string, textBlock);
return boundaries.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
@@ -96,7 +103,14 @@ public class EntityCreationService {
public Set byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
- List boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
+ List boundaries = RedactionSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
+ return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
+ }
+
+
+ public Set byKeyword(String keyword, String type, EntityType entityType, SemanticNode node) {
+
+ List boundaries = RedactionSearchUtils.findBoundariesByString(keyword, node.buildTextBlock());
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
}
@@ -109,10 +123,24 @@ public class EntityCreationService {
}
+ public RedactionEntity byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+ int expandedStart = getExpandedStartByRegex(entity, regexPattern);
+ return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+ }
+
+
+ public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+ int expandedEnd = getExpandedEndByRegex(entity, regexPattern);
+ return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+ }
+
+
public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);
- addEntityToGraph(entity, node.getTableOfContents());
+ addEntityToGraph(entity, node);
return entity;
}
@@ -129,25 +157,65 @@ public class EntityCreationService {
RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(RedactionEntity::isRedaction));
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
+ entitiesToMerge.stream().map(RedactionEntity::getMatchedRules).flatMap(Collection::stream).forEach(mergedEntity::addMatchedRule);
+
RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
- mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
- addEntityToGraph(mergedEntity, node.getTableOfContents());
+ addEntityToGraph(mergedEntity, node);
return mergedEntity;
}
+ public RedactionEntity byTableCellAsHighlight(TableCellNode tableCellNode, String type, EntityType entityType) {
+
+ RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCellNode.getBoundary().start(), tableCellNode.getBoundary().start()),
+ type,
+ entityType);
+
+ String positionId = IdBuilder.buildId(tableCellNode.getBBox().keySet(), tableCellNode.getBBox().values().stream().toList());
+ highlightEntity.setRedactionPositionsPerPage(tableCellNode.getBBox()
+ .entrySet()
+ .stream()
+ .map(entry -> new RedactionPosition(positionId, entry.getKey(), List.of(entry.getValue())))
+ .toList());
+ addEntityToGraph(highlightEntity, tableCellNode);
+ return highlightEntity;
+ }
+
+
public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
}
+ public void addEntityToGraph(RedactionEntity entity, SemanticNode node) {
+
+ TableOfContents tableOfContents = node.getTableOfContents();
+ try {
+ addEntityToGraph(entity, tableOfContents);
+ entity.addIntersectingNode(tableOfContents.getRoot().getNode());
+ tableOfContents.getRoot().getNode().getEntities().add(entity);
+ } catch (NoSuchElementException e) {
+ entity.setDeepestFullyContainingNode(tableOfContents.getRoot().getNode());
+ entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().buildTextBlock());
+ entity.addIntersectingNode(tableOfContents.getRoot().getNode());
+ addToPages(entity);
+ addEntityToNodeEntitySets(entity);
+ }
+ }
+
+
public void addEntityToGraph(RedactionEntity entity, TableOfContents tableOfContents) {
- tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity);
+ SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
+ .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
+ .findFirst()
+ .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
+
+ containingNode.addThisToEntityIfIntersects(entity);
TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
entityEnrichmentService.enrichEntity(entity, textBlock);
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java
index e963f695..f5c5382a 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java
@@ -1,13 +1,11 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import java.util.Arrays;
-import java.util.Collection;
import java.util.List;
import java.util.Objects;
import org.springframework.stereotype.Service;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -29,7 +27,7 @@ public class EntityEnrichmentService {
}
- private CharSequence findTextAfter(int index, TextBlock textBlock) {
+ private String findTextAfter(int index, TextBlock textBlock) {
int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().end());
String textAfter = textBlock.subSequence(index, endOffset).toString();
@@ -44,7 +42,7 @@ public class EntityEnrichmentService {
}
- private CharSequence findTextBefore(int index, TextBlock textBlock) {
+ private String findTextBefore(int index, TextBlock textBlock) {
int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start());
String textBefore = textBlock.subSequence(offsetBefore, index).toString();
@@ -90,27 +88,4 @@ public class EntityEnrichmentService {
return startWithSpace ? " " + result : result;
}
-
- public static void setFields(RedactionEntity redactionEntity, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
- if (redactionEntity.getMatchedRule() == -1 && matchedRule != -1) {
- redactionEntity.setMatchedRule(matchedRule);
- }
- if (redactionEntity.getRedactionReason().equals("") && redactionReason != null) {
- redactionEntity.setRedactionReason(redactionReason);
- }
- if (redactionEntity.getLegalBasis().equals("") && legalBasis != null) {
- redactionEntity.setLegalBasis(legalBasis);
- }
- if (engine != null) {
- redactionEntity.addEngine(engine);
- }
- }
-
-
- public static void setFields(Collection redactionEntities, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
- redactionEntities.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
- }
-
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
index afab6aa3..0929f62d 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
@@ -7,6 +7,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import lombok.RequiredArgsConstructor;
@@ -22,7 +23,7 @@ public class ManualRedactionApplicationService {
RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
.stream()
- .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getValue()))
+ .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
.findFirst()
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
@@ -38,8 +39,9 @@ public class ManualRedactionApplicationService {
entityToBeResized.setResized(true);
entityToBeResized.getBoundary().setStart(newStartOffset);
entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
+ SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getTableOfContents().getRoot().getNode();
entityToBeResized.removeFromGraph();
- entityCreationService.addEntityToGraph(entityToBeResized, entityToBeResized.getDeepestFullyContainingNode().getTableOfContents());
+ entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java
index 6f65d983..31d62e79 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java
@@ -99,6 +99,16 @@ public class RectangleTransformations {
}
+ public static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+ return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
+ (float) rectangle2D.getMaxY(),
+ (float) rectangle2D.getWidth(),
+ (float) rectangle2D.getHeight(),
+ pageNumber);
+ }
+
+
public static String toString(Rectangle2D rectangle2D) {
return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/CharSequenceSearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtils.java
similarity index 51%
rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/CharSequenceSearchUtils.java
rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtils.java
index 9e89f1f5..2037b882 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/CharSequenceSearchUtils.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtils.java
@@ -1,6 +1,6 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
+import static java.lang.String.format;
import java.util.LinkedList;
import java.util.List;
@@ -8,10 +8,11 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
-public class CharSequenceSearchUtils {
+public class RedactionSearchUtils {
public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
@@ -20,6 +21,12 @@ public class CharSequenceSearchUtils {
}
+ public static boolean exactMatch(CharSequence charSequence, String regexPattern) {
+
+ return charSequence.toString().matches(regexPattern);
+ }
+
+
public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
@@ -31,7 +38,49 @@ public class CharSequenceSearchUtils {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
Matcher matcher = pattern.matcher(searchText);
- return new Boundary(matcher.start(), matcher.end());
+ if (matcher.find()) {
+ return new Boundary(matcher.start(), matcher.end());
+ }
+ throw new IllegalArgumentException(format("Charsequence %s does not contain any matches for pattern %s", searchText, regexPattern));
+ }
+
+
+ public static int getExpandedEndByRegex(RedactionEntity entity, String regexPattern) {
+
+ int expandedEnd;
+ if (anyMatch(entity.getTextAfter(), regexPattern)) {
+ Boundary postfixBoundary = findFirstBoundary(regexPattern, entity.getTextAfter());
+ expandedEnd = postfixBoundary.end() + entity.getBoundary().end();
+ } else {
+ expandedEnd = entity.getBoundary().end();
+ }
+ return expandedEnd;
+ }
+
+
+ public static int getExpandedStartByRegex(RedactionEntity entity, String regexPattern) {
+
+ int expandedStart;
+ if (anyMatch(entity.getTextBefore(), regexPattern)) {
+ Boundary prefixBoundary = findFirstBoundary(regexPattern, entity.getTextBefore());
+ expandedStart = prefixBoundary.start() + entity.getBoundary().start() - entity.getTextBefore().length();
+ } else {
+ expandedStart = entity.getBoundary().start();
+ }
+ return expandedStart;
+ }
+
+
+ public static List findBoundariesByRegex(String regexPattern, CharSequence searchText) {
+
+ Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+ Matcher matcher = pattern.matcher(searchText);
+ List boundaries = new LinkedList<>();
+ while (matcher.find()) {
+ boundaries.add(new Boundary(matcher.start(), matcher.end()));
+ }
+ return boundaries;
+
}
@@ -43,7 +92,7 @@ public class CharSequenceSearchUtils {
while (matcher.find()) {
boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
}
- return boundaries.stream().filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
+ return boundaries;
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java
index e56ca4e0..d524e7d5 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java
@@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
import java.util.Comparator;
import java.util.List;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
public class TextPositionOperations {
- public static List mergeAndSortTextPositionSequenceByYThenX(List textBlocks) {
+ public static List mergeAndSortTextPositionSequenceByYThenX(List textBlocks) {
return textBlocks.stream()//
.flatMap(tb -> tb.getSequences().stream())//
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
index 48abdef2..9ca514e8 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
@@ -13,7 +13,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/conditions/SemanticNodeRuleConditions.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/conditions/SemanticNodeRuleConditions.java
new file mode 100644
index 00000000..8509f7bd
--- /dev/null
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/conditions/SemanticNodeRuleConditions.java
@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SemanticNodeRuleConditions {
+
+ public boolean hasEntitiesOfType(SemanticNode semanticNode, String type) {
+
+ return semanticNode.getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+ }
+
+
+ public boolean hasEntitiesOfType(SemanticNode semanticNode, List types) {
+
+ return types.stream().allMatch(type -> hasEntitiesOfType(semanticNode, type));
+ }
+
+}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java
index 3d954036..8d8b9638 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java
@@ -4,7 +4,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -17,7 +17,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class CellValue {
- private List textBlocks = new ArrayList<>();
+ private List textBlocks = new ArrayList<>();
private int rowSpanStart;
@@ -27,9 +27,9 @@ public class CellValue {
StringBuilder sb = new StringBuilder();
- Iterator itty = textBlocks.iterator();
+ Iterator itty = textBlocks.iterator();
while (itty.hasNext()) {
- TextBlock textBlock = itty.next();
+ ClassificationTextBlock textBlock = itty.next();
TextPositionSequence previous = null;
for (TextPositionSequence word : textBlock.getSequences()) {
if (previous != null) {
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
index fa9a3245..94796c3a 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
@@ -23,8 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
import com.iqser.red.service.redaction.v1.model.ArgumentType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@@ -1354,7 +1354,7 @@ public class Section {
entity.setRedactionReason(reason);
entity.setTargetSequences(value.getTextBlocks()
.stream()
- .map(TextBlock::getSequences)
+ .map(ClassificationTextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
entity.setLegalBasis(legalBasis);
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
index fdf54362..7271a5b3 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
@@ -24,7 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
@@ -36,7 +36,9 @@ import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@@ -103,8 +105,9 @@ public class DroolsExecutionService {
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
kieSession.fireAllRules();
-
- return getFileAttributes(kieSession);
+ List resultingFileAttributes = getFileAttributes(kieSession);
+ kieSession.dispose();
+ return resultingFileAttributes;
}
@@ -127,7 +130,7 @@ public class DroolsExecutionService {
List fileAttributes = new LinkedList<>();
QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
for (QueryResultsRow resultsRow : entitiesResult) {
- fileAttributes.add((FileAttribute) resultsRow.get("$result"));
+ fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute"));
}
return fileAttributes;
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
index 7d9b2884..d19caa38 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
@@ -11,9 +11,9 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
@@ -75,7 +75,11 @@ public class RedactionLogCreatorService {
private RedactionLogEntry createRedactionLogEntry(RedactionEntity entity, String dossierTemplateId) {
Set referenceIds = new HashSet<>();
- entity.getReferences().forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
+ entity.getReferences()
+ .stream()
+ .filter(redactionEntity -> !redactionEntity.isRemoved())
+ .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
+ int sectionNumber = entity.getDeepestFullyContainingNode().getTocId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTocId().get(0);
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
@@ -88,7 +92,7 @@ public class RedactionLogCreatorService {
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(entity.getDeepestFullyContainingNode().toString())
- .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
+ .sectionNumber(sectionNumber)
.matchedRule(entity.getMatchedRule())
.isDictionaryEntry(entity.isDictionaryEntry())
.textAfter(entity.getTextAfter().toString())
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
index acf402b9..ea1ec596 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
@@ -15,9 +15,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import lombok.RequiredArgsConstructor;
@@ -52,9 +52,9 @@ public class SectionGridCreatorService {
continue;
}
- if (textBlock instanceof TextBlock) {
+ if (textBlock instanceof ClassificationTextBlock) {
- TextBlock tb = (TextBlock) textBlock;
+ ClassificationTextBlock tb = (ClassificationTextBlock) textBlock;
classifiedDoc.getSectionGrid()
.getRectanglesPerPage()
.computeIfAbsent(page, (x) -> new ArrayList<>())
@@ -65,10 +65,10 @@ public class SectionGridCreatorService {
paragraph.getPageBlocks().size(),
null));
- } else if (textBlock instanceof Table) {
+ } else if (textBlock instanceof TablePageBlock) {
List cellRectangles = new ArrayList<>();
- for (List row : ((Table) textBlock).getRows()) {
+ for (List row : ((TablePageBlock) textBlock).getRows()) {
for (Cell cell : row) {
if (cell != null) {
cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell.getWidth(), (float) cell.getHeight()));
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
index b3c81d02..a95ca514 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
@@ -19,9 +19,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
@@ -42,8 +42,8 @@ public class SectionTextBuilderService {
AtomicInteger sectionNumber = new AtomicInteger(1);
for (Section section : classifiedDoc.getSections()) {
- List tables = section.getTables();
- for (Table table : tables) {
+ List tables = section.getTables();
+ for (TablePageBlock table : tables) {
sectionTexts.addAll(processTablePerRow(table, sectionNumber));
sectionNumber.incrementAndGet();
}
@@ -70,14 +70,14 @@ public class SectionTextBuilderService {
}
- private List processTablePerRow(Table table, AtomicInteger sectionNumber) {
+ private List processTablePerRow(TablePageBlock table, AtomicInteger sectionNumber) {
List sectionTexts = new ArrayList<>();
boolean hasHeader = hasTableHeader(table);
for (List row : table.getRows()) {
- List textBlocks = new ArrayList<>();
+ List textBlocks = new ArrayList<>();
List areas = new ArrayList<>();
Map tabularData = new HashMap<>();
List startOffsets = new ArrayList<>();
@@ -125,13 +125,13 @@ public class SectionTextBuilderService {
}
- public String getRowText(List rowTextBlocks) {
+ public String getRowText(List rowTextBlocks) {
return SearchableText.buildString(rowTextBlocks.stream().map(textBlock -> textBlock.getSequences()).flatMap(List::stream).collect(Collectors.toList()));
}
- private boolean hasTableHeader(Table table) {
+ private boolean hasTableHeader(TablePageBlock table) {
return table.getRows().stream().anyMatch(row -> row.stream().anyMatch(cell -> !cell.isHeaderCell() && !cell.getHeaderCells().isEmpty()));
}
@@ -170,13 +170,13 @@ public class SectionTextBuilderService {
private SectionText processText(SearchableText searchableText,
- List paragraphTextBlocks,
+ List paragraphTextBlocks,
String headline,
AtomicInteger sectionNumber,
List images) {
SectionText sectionText = new SectionText();
- for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
+ for (ClassificationTextBlock paragraphTextBlock : paragraphTextBlocks) {
SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()),
paragraphTextBlock.getWidth(),
paragraphTextBlock.getHeight(),
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
index c0be771c..207f94e5 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
@@ -36,7 +36,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.se
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
@@ -83,7 +83,6 @@ public class AnalyzeService {
ImageServiceResponseAdapter imageServiceResponseAdapter;
ImportedRedactionService importedRedactionService;
SectionFinder sectionFinder;
- DocumentGraphFactory documentGraphFactory;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@@ -112,7 +111,7 @@ public class AnalyzeService {
throw new RedactionException(e);
}
- DocumentGraph documentGraph = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+ DocumentGraph documentGraph = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
@@ -206,8 +205,10 @@ public class AnalyzeService {
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
+ log.debug("Starting Dictionary Search");
+ long dictSearchStart = System.currentTimeMillis();
entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
+ log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
Set addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary,
documentGraph,
kieContainer,
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
index 01463dec..76350520 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
@@ -19,7 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
index 102e2bc1..30b451a9 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
@@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
+import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@@ -10,12 +11,14 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import lombok.AccessLevel;
@@ -39,12 +42,15 @@ public class EntityRedactionService {
AnalyzeRequest analyzeRequest,
List nerEntities) {
+ log.debug("Starting Drools Execution");
+ long droolsStart = System.currentTimeMillis();
List allFileAttributes = droolsExecutionService.executeRules(kieContainer,
documentGraph,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
+ log.debug("Finished Drools Execution in {} ms", System.currentTimeMillis() - droolsStart);
return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
}
@@ -67,17 +73,30 @@ public class EntityRedactionService {
}
- public void addDictionaryEntities(Dictionary dictionary, DocumentGraph documentGraph) {
+ public void addDictionaryEntities(Dictionary dictionary, DocumentGraph document) {
- dictionary.getDictionaryModels().forEach(dictionaryModel -> addDictionaryModelEntities(dictionaryModel, documentGraph));
+ List foundEntities = new LinkedList<>();
+ for (var model : dictionary.getDictionaryModels()) {
+ findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
+ findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
+ findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
+ }
+ foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
}
- private void addDictionaryModelEntities(DictionaryModel dictionaryModel, DocumentGraph documentGraph) {
+ private void findEntitiesWithSearchImplementation(DocumentGraph documentGraph,
+ SearchImplementation searchImplementation,
+ EntityType entityType,
+ List foundEntities,
+ String type) {
- entityCreationService.bySearchImplementation(dictionaryModel.getEntriesSearch(), dictionaryModel.getType(), EntityType.ENTITY, documentGraph);
- entityCreationService.bySearchImplementation(dictionaryModel.getFalsePositiveSearch(), dictionaryModel.getType(), EntityType.FALSE_POSITIVE, documentGraph);
- entityCreationService.bySearchImplementation(dictionaryModel.getFalseRecommendationsSearch(), dictionaryModel.getType(), EntityType.FALSE_RECOMMENDATION, documentGraph);
+ TextBlock textBlock = documentGraph.getTextBlock();
+ searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
+ .stream()
+ .filter(boundary -> entityCreationService.isValidEntityBoundary(textBlock, boundary))
+ .map(bounds -> RedactionEntity.initialEntityNode(bounds, type, entityType))
+ .forEach(foundEntities::add);
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Liszt.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Liszt.java
new file mode 100644
index 00000000..c5503336
--- /dev/null
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Liszt.java
@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.List;
+
+/**
+ * Drools fucked up and introduced a bug with newer versions of java, this is needed or else:
+ * java.lang.IncompatibleClassChangeError: Method 'java.util.List java.util.List.of(java.lang.Object, java.lang.Object)' must be InterfaceMethodref constant
+ * Will be thrown on LHS of rules
+ */
+public class Liszt {
+
+ public static List of(E e1, E e2) {
+
+ return List.of(e1, e2);
+ }
+
+}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java
index 90aa5988..ebd3e6ef 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java
@@ -12,8 +12,8 @@ import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
index 9df2c7cb..e0cdf473 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
@@ -15,8 +15,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@@ -46,12 +46,12 @@ public class PdfVisualisationService {
if (textBlock.getPage() != page) {
continue;
}
- if (textBlock instanceof TextBlock) {
+ if (textBlock instanceof ClassificationTextBlock) {
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
- visualizeTextBlock((TextBlock) textBlock, contentStream);
- } else if (textBlock instanceof Table) {
+ visualizeTextBlock((ClassificationTextBlock) textBlock, contentStream);
+ } else if (textBlock instanceof TablePageBlock) {
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
- visualizeTable((Table) textBlock, contentStream);
+ visualizeTable((TablePageBlock) textBlock, contentStream);
}
}
@@ -75,10 +75,10 @@ public class PdfVisualisationService {
if (textBlock == null) {
continue;
}
- if (textBlock instanceof TextBlock) {
- visualizeTextBlock((TextBlock) textBlock, contentStream);
- } else if (textBlock instanceof Table) {
- visualizeTable((Table) textBlock, contentStream);
+ if (textBlock instanceof ClassificationTextBlock) {
+ visualizeTextBlock((ClassificationTextBlock) textBlock, contentStream);
+ } else if (textBlock instanceof TablePageBlock) {
+ visualizeTable((TablePageBlock) textBlock, contentStream);
}
}
@@ -95,7 +95,7 @@ public class PdfVisualisationService {
}
- private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
+ private void visualizeTextBlock(ClassificationTextBlock textBlock, PDPageContentStream contentStream) throws IOException {
contentStream.setStrokingColor(Color.RED);
@@ -121,7 +121,7 @@ public class PdfVisualisationService {
@SneakyThrows
- private void drawPositions(PDPageContentStream contentStream, TextBlock textBlock) {
+ private void drawPositions(PDPageContentStream contentStream, ClassificationTextBlock textBlock) {
contentStream.setNonStrokingColor(Color.BLUE);
contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
@@ -145,7 +145,7 @@ public class PdfVisualisationService {
}
- private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+ private void visualizeTable(TablePageBlock table, PDPageContentStream contentStream) throws IOException {
for (List| row : table.getRows()) {
for (Cell cell : row) {
@@ -156,7 +156,7 @@ public class PdfVisualisationService {
contentStream.stroke();
contentStream.setStrokingColor(Color.GREEN);
- for (TextBlock textBlock : cell.getTextBlocks()) {
+ for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
contentStream.addRect(textBlock.getPdfMinX(),
textBlock.getPdfMinY(),
textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
index 4cdf09c0..287309ee 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
@@ -162,7 +162,8 @@ public abstract class AbstractRedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false));
- when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, false));
+ when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
+ false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_PII, false));
when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY_INDICATOR, false));
@@ -172,8 +173,10 @@ public abstract class AbstractRedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(SIGNATURE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false));
- when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,true));
- when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,true));
+ when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,
+ true));
+ when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
+ true));
}
@@ -237,7 +240,7 @@ public abstract class AbstractRedactionIntegrationTest {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
if (resource == null) {
- throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+ throw new IllegalArgumentException("could not load classpath resource: drools/prod_rules_new.drl");
}
List stringList = Files.readAllLines(new File(resource.getPath()).toPath());
return String.join("\n", stringList);
@@ -426,22 +429,30 @@ public abstract class AbstractRedactionIntegrationTest {
@SneakyThrows
protected AnalyzeRequest uploadFileToStorage(String file) {
- return prepareStorage(file, "files/cv_service_empty_response.json");
+ return prepareStorage(file, "files/cv_service_empty_response.json", "files/empty_image_response.json");
}
@SneakyThrows
protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
- ClassPathResource pdfFileResource = new ClassPathResource(file);
- ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-
- return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+ return prepareStorage(file, cvServiceResponseFile, "files/empty_image_response.json");
}
@SneakyThrows
- protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+ protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile, String imageServiceResponseFile) {
+
+ ClassPathResource pdfFileResource = new ClassPathResource(file);
+ ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+ ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponseFile);
+
+ return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageServiceResponseFileResource.getInputStream());
+ }
+
+
+ @SneakyThrows
+ protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageServiceResponseStream) {
AnalyzeRequest request = AnalyzeRequest.builder()
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
@@ -450,10 +461,13 @@ public abstract class AbstractRedactionIntegrationTest {
.lastProcessed(OffsetDateTime.now())
.build();
+ storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
cvServiceResponseFileStream);
- storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+ storageService.storeObject(TenantContext.getTenantId(),
+ RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+ imageServiceResponseStream);
return request;
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java
deleted file mode 100644
index 49db5bed..00000000
--- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java
+++ /dev/null
@@ -1,546 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.kie.api.KieServices;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.KieFileSystem;
-import org.kie.api.builder.KieModule;
-import org.kie.api.builder.KieRepository;
-import org.kie.api.builder.ReleaseId;
-import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
-import org.mockito.stubbing.Answer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(AbstractTestWithDictionaries.TestConfiguration.class)
-public class AbstractTestWithDictionaries {
-
- protected static final String RULES = loadFromClassPath("drools/testRules.drl");
- protected static final String RULES_PATH = "drools/rules.drl";
- protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
- protected static final String MERGE_ENTITY_RULES_PATH = "drools/testRules.drl";
- private static final String VERTEBRATE = "vertebrate";
- private static final String ADDRESS = "CBI_address";
- private static final String AUTHOR = "CBI_author";
- private static final String SPONSOR = "CBI_sponsor";
- private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
- private static final String REDACTION_INDICATOR = "redaction_indicator";
- private static final String HINT_ONLY = "hint_only";
- private static final String MUST_REDACT = "must_redact";
- private static final String PUBLISHED_INFORMATION = "published_information";
- private static final String TEST_METHOD = "test_method";
- private static final String PURITY = "purity";
- private static final String IMAGE = "image";
- private static final String LOGO = "logo";
- private static final String SIGNATURE = "signature";
- private static final String FORMULA = "formula";
- private static final String OCR = "ocr";
- private static final String DOSSIER_REDACTIONS = "dossier_redactions";
- private static final String IMPORTED_REDACTION = "imported_redaction";
- private static final String PII = "PII";
- private static final String ROTATE_SIMPLE = "RotateSimple";
-
- protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
- protected final static String TEST_DOSSIER_ID = "123";
- protected final static String TEST_FILE_ID = "123";
-
- @Autowired
- protected StorageService storageService;
-
- @MockBean
- protected DictionaryClient dictionaryClient;
-
- @MockBean
- protected RulesClient rulesClient;
-
- private final Map> dictionary = new HashMap<>();
- private final Map> dossierDictionary = new HashMap<>();
- private final Map> falsePositive = new HashMap<>();
- private final Map> falseRecommendation = new HashMap<>();
- private final Map typeColorMap = new HashMap<>();
- private final Map hintTypeMap = new HashMap<>();
- private final Map caseInSensitiveMap = new HashMap<>();
- private final Map recommendationTypeMap = new HashMap<>();
- private final Map rankTypeMap = new HashMap<>();
- private final Colors colors = new Colors();
- private final Map reanalysisVersions = new HashMap<>();
- private final Set deleted = new HashSet<>();
-
-
- @BeforeEach
- public void stubClients() {
-
- TenantContext.setTenantId("redaction");
-
- when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
- when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
-
- loadDictionaryForTest();
- loadTypeForTest();
- loadNerForTest();
- when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
- when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
- when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
- when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
- .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
- .type(DOSSIER_REDACTIONS)
- .dossierTemplateId(TEST_DOSSIER_ID)
- .hexColor("#ffe187")
- .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
- .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
- .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
- .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
- .build()));
-
- mockDictionaryCalls(null);
- mockDictionaryCalls(0L);
-
- when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
- }
-
-
- private void mockDictionaryCalls(Long version) {
-
- when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE,
- false));
- when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ADDRESS, false));
- when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(AUTHOR, false));
- when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SPONSOR, false));
- when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(
- NO_REDACTION_INDICATOR,
- false));
- when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(
- REDACTION_INDICATOR,
- false));
- when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY, false));
- when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT,
- false));
- when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(
- PUBLISHED_INFORMATION,
- false));
- when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD,
- false));
- when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PII, false));
- when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY, false));
- when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE, false));
- when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(OCR, false));
- when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO, false));
- when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE, false));
- when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA, false));
- when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
- false));
- when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(
- DOSSIER_REDACTIONS,
- true));
- when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer | | | | | | | | | | | |