diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java index e3073231..08f8920b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java @@ -7,10 +7,20 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.AllArgsConstructor; @@ -29,6 +39,7 @@ public class SectionText { private boolean isTable; private String headline; + List paragraphs; @Builder.Default private List sectionAreas = new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java index 3061541b..0a9f62d4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java @@ -26,6 +26,8 @@ public class TextBlock extends AbstractTextContainer { @JsonIgnore private int rotation; + private int indexOnPage; + @JsonIgnore private String mostPopularWordFont; @@ -174,8 +176,8 @@ public class TextBlock extends AbstractTextContainer { } - public TextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation) { - + public TextBlock(float minX, float maxX, float minY, float maxY, List sequences, int rotation, int indexOnPage) { + this.indexOnPage = indexOnPage; this.minX = minX; this.maxX = maxX; this.minY = minY; @@ -238,7 +240,7 @@ public class TextBlock extends AbstractTextContainer { public TextBlock copy() { - return new TextBlock(minX, maxX, minY, maxY, sequences, rotation); + return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java index 2cff711a..9f6ec4f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java @@ -30,13 +30,15 @@ public class BlockificationService { * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions. * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this! * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling. - * @param textPositions The words of a page. + * + * @param textPositions The words of a page. * @param horizontalRulingLines Horizontal table lines. - * @param verticalRulingLines Vertical table lines. + * @param verticalRulingLines Vertical table lines. * @return Page object that contains the Textblock and text statistics. */ public Page blockify(List textPositions, List horizontalRulingLines, List verticalRulingLines) { + int indexOnPage = 0; List chunkWords = new ArrayList<>(); List chunkBlockList1 = new ArrayList<>(); @@ -62,7 +64,9 @@ public class BlockificationService { prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation(); } - TextBlock cb1 = buildTextBlock(chunkWords); + TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); + indexOnPage++; + chunkBlockList1.add(cb1); chunkWords = new ArrayList<>(); @@ -102,7 +106,7 @@ public class BlockificationService { } } - TextBlock cb1 = buildTextBlock(chunkWords); + TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); if (cb1 != null) { chunkBlockList1.add(cb1); } @@ -163,7 +167,7 @@ public class BlockificationService { } - private TextBlock buildTextBlock(List wordBlockList) { + private TextBlock buildTextBlock(List wordBlockList, int indexOnPage) { TextBlock textBlock = null; @@ -182,7 +186,13 @@ public class BlockificationService { styleFrequencyCounter.add(wordBlock.getFontStyle()); if (textBlock == null) { - textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation()); + textBlock = new TextBlock(wordBlock.getMinXDirAdj(), + wordBlock.getMaxXDirAdj(), + wordBlock.getMinYDirAdj(), + wordBlock.getMaxYDirAdj(), + wordBlockList, + wordBlock.getRotation(), + indexOnPage); } else { TextBlock spatialEntity = textBlock.union(wordBlock); textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight()); @@ -213,10 +223,38 @@ public class BlockificationService { List horizontalRulingLines, List verticalRulingLines) { - return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) // - || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) // - || isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) // - || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); // + return isSplitByRuling(maxX, + minY, + word.getMinXDirAdj(), + word.getMinYDirAdj(), + verticalRulingLines, + word.getDir().getDegrees(), + word.getPageWidth(), + word.getPageHeight()) // + || isSplitByRuling(minX, + minY, + word.getMinXDirAdj(), + word.getMaxYDirAdj(), + horizontalRulingLines, + word.getDir().getDegrees(), + word.getPageWidth(), + word.getPageHeight()) // + || isSplitByRuling(maxX, + minY, + word.getMinXDirAdj(), + word.getMinYDirAdj(), + horizontalRulingLines, + word.getDir().getDegrees(), + word.getPageWidth(), + word.getPageHeight()) // + || isSplitByRuling(minX, + minY, + word.getMinXDirAdj(), + word.getMaxYDirAdj(), + verticalRulingLines, + word.getDir().getDegrees(), + word.getPageWidth(), + word.getPageHeight()); // } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java new file mode 100644 index 00000000..18c6fbb8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java @@ -0,0 +1,96 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.awt.geom.Rectangle2D; +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class AtomicTextBlockEntity implements CharSequence { + + Integer id; + //string coords + Integer offset; + String searchText; + List lineBreaks; + + //position coords + List stringIdxToPositionIdx; + List positions; + + Node parent; + + + public int indexOf(String searchTerm) { + + int pos = searchText.indexOf(searchTerm); + return pos == -1 ? -1 : pos + offset; + } + + + public int numberOfLines() { + + return lineBreaks.size(); + } + + + public int getNextLinebreak(int startIndex) { + + return lineBreaks.stream()// + .filter(linebreak -> linebreak > startIndex) // + .findFirst() // + .orElse(searchText.length()) + offset; + } + + + public Rectangle2D getPosition(int stringIdx) { + + return positions.get(stringIdxToPositionIdx.get(stringIdx - offset)); + } + + + public List getPositions(int startStringIdx, int endStringIdx) { + + return positions.subList(startStringIdx - offset, endStringIdx - offset); + } + + + @Override + public int length() { + + return searchText.length(); + } + + + @Override + public char charAt(int index) { + + return searchText.charAt(index - offset); + } + + + @Override + public CharSequence subSequence(int start, int end) { + + throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!"); + } + + + public String getFirstLine() { + + return searchText.substring(0, getNextLinebreak(0) - offset); + } + + @Override + public String toString() { + return searchText; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java new file mode 100644 index 00000000..f35b0109 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; +import java.util.stream.Stream; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentEntity implements Node { + + List sections; + List pages; + TableOfContents tableOfContents; + Integer numberOfPages; + + + @Override + public TextBlockEntity buildTextBlock() { + + return streamAllNodes() + .filter(Node::hasTextBlock) + .map(Node::getTextBlock) + .collect(TextBlockCollector.collect()); + } + + + private Stream streamAllNodes() { + + return Stream.concat(Stream.concat(// + tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), // + pages.stream().map(PageEntity::getHeader)), // + pages.stream().map(PageEntity::getFooter)); // + } + + + @Override + public String toString() { + + return tableOfContents.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java new file mode 100644 index 00000000..a5ce23c9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.awt.geom.Rectangle2D; +import java.util.List; + +public class Entity2 { + + String uid; + String value; + List positions; + Boolean redact; + + List neighbours; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java new file mode 100644 index 00000000..6a3e9ce2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java @@ -0,0 +1,47 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FooterEntity implements Node { + + PageEntity page; + TextBlockEntity textBlock; + List entities; + + + @Override + public TextBlockEntity buildTextBlock() { + + return getTextBlock(); + } + + + @Override + public boolean hasTextBlock() { + + return true; + } + + + @Override + public TextBlockEntity getTextBlock() { + + return textBlock; + } + + @Override + public String toString() { + return textBlock.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java new file mode 100644 index 00000000..7f07fc42 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java @@ -0,0 +1,47 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class HeaderEntity implements Node { + + PageEntity page; + TextBlockEntity textBlock; + List entities; + + + @Override + public TextBlockEntity buildTextBlock() { + + return getTextBlock(); + } + + + @Override + public boolean hasTextBlock() { + + return true; + } + + + @Override + public TextBlockEntity getTextBlock() { + + return textBlock; + } + + @Override + public String toString() { + return textBlock.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java new file mode 100644 index 00000000..631dabfa --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +public interface Node { + TextBlockEntity buildTextBlock(); + + + default boolean hasTextBlock() { + + return false; + } + + + default TextBlockEntity getTextBlock() { + + throw new UnsupportedOperationException("Only terminal Nodes have direct access to TextBlocks"); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java new file mode 100644 index 00000000..62c154a8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class PageEntity implements Node { + + Integer number; + Integer height; + Integer width; + List sections; + List paragraphs; + HeaderEntity header; + FooterEntity footer; + + + @Override + public TextBlockEntity buildTextBlock() { + + return null; + } + + @Override + public String toString() { + return number.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java new file mode 100644 index 00000000..2fad58b0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java @@ -0,0 +1,54 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ParagraphEntity implements Node { + + String tocId; + Integer id; + Integer numberOnPage; + Integer numberInSection; + + //foreign key + SectionEntity parentSection; + PageEntity page; + TextBlockEntity textBlock; + List entities; + + + @Override + public TextBlockEntity buildTextBlock() { + + return getTextBlock(); + } + + + @Override + public boolean hasTextBlock() { + + return true; + } + + + @Override + public TextBlockEntity getTextBlock() { + + return textBlock; + } + + + @Override + public String toString() { + return textBlock.toString(); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java new file mode 100644 index 00000000..398e81a0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class SectionEntity implements Node { + + Integer id; + TableOfContents tableOfContents; + String tocId; + Node parent; + //foreign key + TextBlockEntity headline; + List subSections; + List paragraphs; + List pages; + List entities; + + + @Override + public String toString() { + + return tocId + ": " + headline.toString(); + } + + + @Override + public TextBlockEntity buildTextBlock() { + + return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node).map(Node::getTextBlock).collect(TextBlockCollector.collect()); + } + + + @Override + public boolean hasTextBlock() { + + return true; + } + + + @Override + public TextBlockEntity getTextBlock() { + + return headline; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java new file mode 100644 index 00000000..50b69f94 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class TableCellEntity implements Node { + + //foreign key + Integer table; + TextBlockEntity textBlock; + List entities; + + + @Override + public TextBlockEntity buildTextBlock() { + + return textBlock; + } + + + @Override + public boolean hasTextBlock() { + + return true; + } + + + @Override + public TextBlockEntity getTextBlock() { + + return textBlock; + } + + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java new file mode 100644 index 00000000..1d7c4ad2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java @@ -0,0 +1,53 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.List; +import java.util.stream.Stream; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class TableEntity implements Node { + + Integer id; + Integer numberOfRows; + Integer numberOfCols; + List> tableCells; + + // graph connection + SectionEntity section; + PageEntity page; + List entities; + + + private Stream streamTableCells() { + + return tableCells.stream().flatMap(List::stream); + } + + + private Stream streamTableRow(int row) { + + return tableCells.get(row).stream(); + } + + + private Stream streamTableCol(int col) { + + return tableCells.stream().map(row -> row.get(col)); + } + + + @Override + public TextBlockEntity buildTextBlock() { + + return streamTableCells().map(TableCellEntity::buildTextBlock).collect(TextBlockCollector.collect()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java new file mode 100644 index 00000000..30e5d5c1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -0,0 +1,109 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static java.lang.String.format; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import javax.management.openmbean.InvalidKeyException; + +import lombok.Data; + +@Data +public class TableOfContents { + + public static final String SECTION = "Sec"; + public static final String HEADLINE = "H"; + public static final String PARAGRAPH = "Par"; + public static final String TABLE = "Tab"; + + List entries; + + + public TableOfContents() { + + entries = new LinkedList<>(); + } + + + public String createNewEntryAndReturnId(String keyword, String summary, Node node) { + + String id = String.format("%d", entries.size()); + entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node)); + return id; + } + + + public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, Node node) { + + Entry parent = getEntryById(parentId); + String childId = parentId + String.format(".%d", parent.children().size()); + parent.children().add(new Entry(keyword, childId, summary, new LinkedList<>(), node)); + return childId; + } + + + public Entry getEntryById(String parentId) { + + List ids = getIds(parentId); + if (ids.size() < 1) { + throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", parentId)); + } + Entry entry = entries.get(ids.get(0)); + for (int id : ids.subList(1, ids.size())) { + entry = entry.children().get(id); + } + return entry; + } + + + @Override + public String toString() { + + return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList()); + } + + + public String toString(String id) { + + return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList()); + } + + + public Stream streamEntriesInOrder() { + + return entries.stream().flatMap(TableOfContents::flatten); + } + + + public Stream streamSubEntriesInOrder(String parentId) { + + return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten); + } + + + private static List getIds(String idsAsString) { + + return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList(); + } + + + private static Stream flatten(Entry entry) { + + return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten)); + } + + + public record Entry(String type, String id, String summary, List children, Node node) { + + @Override + public String toString() { + + return id + ": " + type + ".: " + summary; + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java new file mode 100644 index 00000000..dad82cf2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java @@ -0,0 +1,53 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.util.Collections; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +public class TextBlockCollector implements Collector { + + public static Collector collect() { + + return new TextBlockCollector(); + } + + + @Override + public Supplier supplier() { + + return new TextBlockEntity(Collections.emptyList()); + } + + + @Override + public BiConsumer accumulator() { + + return TextBlockEntity::concat; + } + + + @Override + public BinaryOperator combiner() { + + return TextBlockEntity::concat; + } + + + @Override + public Function finisher() { + + return Function.identity(); + } + + + @Override + public Set characteristics() { + + return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java new file mode 100644 index 00000000..27086b1c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java @@ -0,0 +1,164 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import java.awt.geom.Rectangle2D; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +public class TextBlockEntity implements CharSequence, Supplier { + + List atomicTextBlocks; + StringBuilder searchTextBuilder; + + //string coords + Integer offset; + + + public TextBlockEntity(List atomicTextBlocks) { + + this.atomicTextBlocks = new LinkedList<>(); + this.searchTextBuilder = new StringBuilder(); + if (atomicTextBlocks.isEmpty()) { + offset = -1; + return; + } + var firstTextBlock = atomicTextBlocks.get(0); + this.atomicTextBlocks.add(firstTextBlock); + this.offset = firstTextBlock.getOffset(); + this.searchTextBuilder.append(firstTextBlock.getSearchText()); + + atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat); + } + + + public TextBlockEntity concat(TextBlockEntity textBlock) { + + if (this.atomicTextBlocks.isEmpty() && this.offset == -1) { + this.offset = textBlock.getOffset(); + } else if (textBlock.getOffset() != offset + length()) { + throw new UnsupportedOperationException("Can only concat consecutive TextBlocks"); + } + + this.searchTextBuilder.append(textBlock.getSearchTextBuilder()); + this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks()); + return this; + } + + + public void concat(AtomicTextBlockEntity atomicTextBlock) { + + if (this.atomicTextBlocks.isEmpty() && this.offset == -1) { + this.offset = atomicTextBlock.getOffset(); + } else if (atomicTextBlock.getOffset() != offset + length()) { + throw new UnsupportedOperationException("Can only concat consecutive TextBlocks"); + } + + this.searchTextBuilder.append(atomicTextBlock.getSearchText()); + this.atomicTextBlocks.add(atomicTextBlock); + } + + + public int indexOf(String searchTerm) { + + int pos = this.searchTextBuilder.indexOf(searchTerm); + return pos == -1 ? -1 : pos + offset; + } + + + public int numberOfLines() { + + return atomicTextBlocks.stream().map(AtomicTextBlockEntity::getLineBreaks).mapToInt(List::size).sum(); + } + + + public int getNextLinebreak(int startIndex) { + + return getAtomicTextBlockByStringIndex(startIndex).getNextLinebreak(startIndex); + } + + + public Rectangle2D getPositions(int stringIdx) { + + return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx); + } + + + public List getPositions(int startStringIdx, int endStringIdx) { + + List textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(startStringIdx, endStringIdx); + + if (textBlocks.size() == 1) { + return textBlocks.get(0).getPositions(startStringIdx, endStringIdx); + } + + var firstTextBlock = textBlocks.get(0); + List positions = new LinkedList<>(firstTextBlock.getPositions(startStringIdx, firstTextBlock.getOffset() + firstTextBlock.length())); + + for (var textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { + positions.addAll(textBlock.getPositions()); + } + + var lastTextBlock = textBlocks.get(textBlocks.size() - 1); + positions.addAll(lastTextBlock.getPositions(lastTextBlock.getOffset(), endStringIdx)); + + return positions; + } + + + private AtomicTextBlockEntity getAtomicTextBlockByStringIndex(int stringIdx) { + + return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getOffset() + textBlock.length()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new); + } + + + private List getAllAtomicTextBlocksPartiallyInStringIdxRange(int startStringIdx, int endStringIdx) { + + if (offset > endStringIdx || offset + length() <= startStringIdx) { + throw new IndexOutOfBoundsException(); + } + + return atomicTextBlocks.stream().filter(tb -> tb.getOffset() <= endStringIdx && tb.getOffset() + tb.length() > startStringIdx).toList(); + } + + + public String getSearchText() { + + return searchTextBuilder.toString(); + } + + + @Override + public int length() { + + return this.searchTextBuilder.length(); + } + + + @Override + public char charAt(int index) { + + return searchTextBuilder.charAt(index); + } + + + @Override + public CharSequence subSequence(int start, int end) { + + throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!"); + } + + + @Override + public TextBlockEntity get() { + + return this; + } + +} + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java new file mode 100644 index 00000000..737f7f58 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java @@ -0,0 +1,209 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.classification.model.Document; +import com.iqser.red.service.redaction.v1.server.classification.model.Footer; +import com.iqser.red.service.redaction.v1.server.classification.model.Header; +import com.iqser.red.service.redaction.v1.server.classification.model.Page; +import com.iqser.red.service.redaction.v1.server.classification.model.Section; +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.AtomicTextBlockEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.FooterEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.HeaderEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.Node; +import com.iqser.red.service.redaction.v1.server.document.graph.PageEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.ParagraphEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.SectionEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; +import com.iqser.red.service.redaction.v1.server.redaction.service.SearchTextWithTextPositionFactory; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class DocumentGraphFactory { + + private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory; + + private Context context; + + + public DocumentEntity buildEntityFromDocument(Document document) { + + context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicInteger(0)); + + context.pages.addAll(document.getPages().stream().map(this::buildPage).toList()); + + // is tracked by Table of Contents + for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) { + addSection(document.getSections().get(sectionIdx), sectionIdx); + } + + // not tracked by Table of Contents + document.getHeaders().stream().map(Header::getTextBlocks).flatMap(List::stream).forEach(this::addHeader); + document.getFooters().stream().map(Footer::getTextBlocks).flatMap(List::stream).forEach(this::addFooter); + + return DocumentEntity.builder().numberOfPages(context.pages.size()).pages(context.pages).sections(context.sections).tableOfContents(context.tableOfContents).build(); + } + + + private void addSection(Section section, int sectionIdx) { + + SectionEntity sectionEntity = SectionEntity.builder() + .id(sectionIdx) + .entities(new LinkedList<>()) + .pages(new LinkedList<>()) + .paragraphs(new LinkedList<>()) + .tableOfContents(context.tableOfContents()) + .subSections(new LinkedList<>()) + .build(); + context.sections().add(sectionEntity); + if (section.getPageBlocks().get(0) instanceof TextBlock) { + sectionEntity.setHeadline(new TextBlockEntity(List.of(buildAtomicTextBlock((TextBlock) section.getPageBlocks().get(0), sectionEntity)))); + section.getPageBlocks().remove(0); + } else { + sectionEntity.setHeadline(emptyTextBlock(sectionEntity)); + } + + String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION, + buildSummary(sectionEntity.getHeadline().getAtomicTextBlocks().get(0)), + sectionEntity); + sectionEntity.setTocId(sectionId); + + int paragraphIdx = 0; + for (AbstractTextContainer abstractTextContainer : section.getPageBlocks()) { + if (abstractTextContainer instanceof TextBlock) { + addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx); + paragraphIdx++; + } + } + } + + + private TextBlockEntity emptyTextBlock(Node parent) { + + return new TextBlockEntity(List.of(AtomicTextBlockEntity.builder() + .id(context.textBlockIdx.getAndIncrement()) + .offset(context.stringOffset.get()) + .searchText("") + .lineBreaks(Collections.emptyList()) + .stringIdxToPositionIdx(Collections.emptyList()) + .positions(Collections.emptyList()) + .parent(parent) + .build())); + } + + + private void addParagraph(SectionEntity sectionEntity, TextBlock originalTextBlock, int idx) { + + PageEntity page = getPage(originalTextBlock.getPage()); + ParagraphEntity paragraph = ParagraphEntity.builder().id(idx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); + + sectionEntity.getParagraphs().add(paragraph); + page.getParagraphs().add(paragraph); + + if (!page.getSections().contains(sectionEntity)) { + page.getSections().add(sectionEntity); + } + + var textBlock = buildAtomicTextBlock(originalTextBlock, paragraph); + paragraph.setTextBlock(new TextBlockEntity(List.of(textBlock))); + + String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph); + paragraph.setTocId(tocId); + } + + + private static String buildSummary(AtomicTextBlockEntity textBlock) { + + if (textBlock == null) { + return " probably a table"; + } + + String[] words = textBlock.getFirstLine().split(" "); + int bound = Math.min(words.length, 4); + List list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); + + return String.join(" ", list); + } + + + private PageEntity buildPage(Page p) { + + return PageEntity.builder() + .height((int) p.getPageHeight()) + .width((int) p.getPageWidth()) + .number(p.getPageNumber()) + .paragraphs(new LinkedList<>()) + .sections(new LinkedList<>()) + .build(); + } + + + private void addFooter(TextBlock textBlock) { + + PageEntity page = getPage(textBlock.getPage()); + FooterEntity footer = FooterEntity.builder().page(page).build(); + AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, footer); + footer.setTextBlock(new TextBlockEntity(List.of(textBlockEntity))); + page.setFooter(footer); + } + + + public void addHeader(TextBlock textBlock) { + + PageEntity page = getPage(textBlock.getPage()); + HeaderEntity header = HeaderEntity.builder().page(page).build(); + AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, header); + header.setTextBlock(new TextBlockEntity(List.of(textBlockEntity))); + page.setHeader(header); + } + + + public AtomicTextBlockEntity buildAtomicTextBlock(TextBlock textBlock, Node parent) { + + SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(textBlock.getSequences()); + int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); + + return AtomicTextBlockEntity.builder() + .id(context.textBlockIdx.getAndIncrement()) + .parent(parent) + .searchText(searchTextWithTextPositionModel.getSearchText()) + .lineBreaks(searchTextWithTextPositionModel.getLineBreaks()) + .positions(searchTextWithTextPositionModel.getPositions()) + .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords()) + .offset(offset) + .build(); + } + + + public PageEntity getPage(int i) { + + return context.pages.stream().filter(page -> page.getNumber() == i).findFirst().orElseThrow(NoSuchFieldError::new); + } + + + record Context( + TableOfContents tableOfContents, + List pages, + List sections, + List headers, + List footers, + AtomicInteger stringOffset, + AtomicInteger textBlockIdx) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java index c849da04..452166b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java @@ -1,9 +1,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import java.awt.geom.Rectangle2D; import java.util.List; -import java.util.Map; - -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import lombok.AccessLevel; import lombok.Builder; @@ -16,7 +14,7 @@ import lombok.experimental.FieldDefaults; public class SearchTextWithTextPositionModel { String searchText; - Map stringCoordsToTextPositionCoords; - List lineBreaksStringCoords; - List textPositionSequences; + List lineBreaks; + List stringCoordsToPositionCoords; + List positions; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 7e68a938..d27f2ff8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1292,7 +1292,7 @@ public class Section { } - private Set findEntities(String value, + public Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java index a059a63a..86b59bc5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java @@ -1,13 +1,13 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.HashMap; +import java.awt.geom.Rectangle2D; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Objects; import org.springframework.stereotype.Service; +import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; @@ -16,7 +16,7 @@ public class SearchTextWithTextPositionFactory { public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List sequences) { - Map stringIdxToPositionIdx = new HashMap<>(); + List stringIdxToPositionIdx = new LinkedList<>(); List lineBreaksStringIdx = new LinkedList<>(); StringBuilder sb = new StringBuilder(); @@ -37,7 +37,7 @@ public class SearchTextWithTextPositionFactory { !isHyphenLinebreak(currentUnicode)) { sb.append(currentUnicode); - stringIdxToPositionIdx.put(stringIdx, positionIdx); + stringIdxToPositionIdx.add(positionIdx); ++stringIdx; } @@ -47,15 +47,24 @@ public class SearchTextWithTextPositionFactory { previousUnicode = " "; sb.append(previousUnicode); - stringIdxToPositionIdx.put(stringIdx, positionIdx); + stringIdxToPositionIdx.add(positionIdx); ++stringIdx; } + assert sb.length() == stringIdxToPositionIdx.size(); + + List positions = sequences.stream()// + .map(TextPositionSequence::getTextPositions)// + .flatMap(List::stream)// + .map(RedTextPosition::getPosition)// + .map(a -> (Rectangle2D) new Rectangle2D.Float(a[0], a[1], a[2], a[3])) + .toList(); + return SearchTextWithTextPositionModel.builder() - .searchText(sb.toString().stripTrailing()) - .lineBreaksStringCoords(lineBreaksStringIdx) - .stringCoordsToTextPositionCoords(stringIdxToPositionIdx) - .textPositionSequences(sequences) + .searchText(sb.toString()) + .lineBreaks(lineBreaksStringIdx) + .stringCoordsToPositionCoords(stringIdxToPositionIdx) + .positions(positions) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java index 4d24f31d..66f53bbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java @@ -16,8 +16,8 @@ public final class OffsetStringUtils { * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only. * * @param str – the String containing the substrings, null returns null, empty returns empty - * @param open – the String identifying the start of the substring, empty returns null - * @param close – the String identifying the end of the substring, empty returns null + * @param open – the String identifying the startIdx of the substring, empty returns null + * @param close – the String identifying the endIdx of the substring, empty returns null * @return a list of Strings with their offsets */ public List substringsBetween(final String str, final String open, final String close) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java index c1270672..336d4fa8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java @@ -73,7 +73,18 @@ public class SearchImplementation { } else { return this.trie.containsMatch(textToCheck); } + } + public List getMatches(CharSequence text) { + if (this.values.isEmpty()) { + return new ArrayList<>(); + } + if (this.pattern != null) { + + return this.pattern.matcher(text).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList()); + } else { + return this.trie.parseText(text).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java new file mode 100644 index 00000000..fb9b6cc9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -0,0 +1,520 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.kie.api.KieServices; +import org.kie.api.builder.KieBuilder; +import org.kie.api.builder.KieFileSystem; +import org.kie.api.builder.KieModule; +import org.kie.api.runtime.KieContainer; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.junit4.SpringRunner; + +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; +import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; + +import lombok.SneakyThrows; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(AbstractTestWithDictionaries.TestConfiguration.class) +public class AbstractTestWithDictionaries { + + private static final String RULES = loadFromClassPath("drools/rules.drl"); + private static final String VERTEBRATE = "vertebrate"; + private static final String ADDRESS = "CBI_address"; + private static final String AUTHOR = "CBI_author"; + private static final String SPONSOR = "CBI_sponsor"; + private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; + private static final String REDACTION_INDICATOR = "redaction_indicator"; + private static final String HINT_ONLY = "hint_only"; + private static final String MUST_REDACT = "must_redact"; + private static final String PUBLISHED_INFORMATION = "published_information"; + private static final String TEST_METHOD = "test_method"; + private static final String PURITY = "purity"; + private static final String IMAGE = "image"; + private static final String LOGO = "logo"; + private static final String SIGNATURE = "signature"; + private static final String FORMULA = "formula"; + private static final String OCR = "ocr"; + private static final String DOSSIER_REDACTIONS = "dossier_redactions"; + private static final String IMPORTED_REDACTION = "imported_redaction"; + private static final String PII = "PII"; + private static final String ROTATE_SIMPLE = "RotateSimple"; + + protected final static String TEST_DOSSIER_TEMPLATE_ID = "123"; + protected final static String TEST_DOSSIER_ID = "123"; + protected final static String TEST_FILE_ID = "123"; + + @Autowired + private StorageService storageService; + + @MockBean + private DictionaryClient dictionaryClient; + + @MockBean + private RulesClient rulesClient; + + private final Map> dictionary = new HashMap<>(); + private final Map> dossierDictionary = new HashMap<>(); + private final Map> falsePositive = new HashMap<>(); + private final Map> falseRecommendation = new HashMap<>(); + private final Map typeColorMap = new HashMap<>(); + private final Map hintTypeMap = new HashMap<>(); + private final Map caseInSensitiveMap = new HashMap<>(); + private final Map recommendationTypeMap = new HashMap<>(); + private final Map rankTypeMap = new HashMap<>(); + private final Colors colors = new Colors(); + private final Map reanalysisVersions = new HashMap<>(); + private final Set deleted = new HashSet<>(); + + + @Before + public void stubClients() { + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) + .build())); + + mockDictionaryCalls(null); + mockDictionaryCalls(0L); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + private static String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader().getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str).append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + } + + + private void mockDictionaryCalls(Long version) { + + when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, + false)); + when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ADDRESS, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + NO_REDACTION_INDICATOR, + false)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + REDACTION_INDICATOR, + false)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT, + false)); + when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + PUBLISHED_INFORMATION, + false)); + when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD, + false)); + when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PII, false)); + when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY, false)); + when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMAGE, false)); + when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(OCR, false)); + when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(LOGO, false)); + when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(SIGNATURE, false)); + when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA, false)); + when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE, + false)); + when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + DOSSIER_REDACTIONS, + true)); + when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + IMPORTED_REDACTION, + true)); + + } + + + private String cleanDictionaryEntry(String entry) { + + return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); + } + + + private List toDictionaryEntry(List entries) { + + if (entries == null) { + entries = Collections.emptyList(); + } + + List dictionaryEntries = new ArrayList<>(); + entries.forEach(entry -> { + dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build()); + }); + return dictionaryEntries; + } + + + private Type getDictionaryResponse(String type, boolean isDossierDictionary) { + + return Type.builder() + .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColorMap.get(type)) + .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type))) + .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>()) + .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>()) + .isHint(hintTypeMap.get(type)) + .isCaseInsensitive(caseInSensitiveMap.get(type)) + .isRecommendation(recommendationTypeMap.get(type)) + .rank(rankTypeMap.get(type)) + .build(); + } + + + private void loadTypeForTest() { + + typeColorMap.put(VERTEBRATE, "#ff85f7"); + typeColorMap.put(ADDRESS, "#ffe187"); + typeColorMap.put(AUTHOR, "#ffe187"); + typeColorMap.put(SPONSOR, "#85ebff"); + typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff"); + typeColorMap.put(REDACTION_INDICATOR, "#caff85"); + typeColorMap.put(HINT_ONLY, "#abc0c4"); + typeColorMap.put(MUST_REDACT, "#fab4c0"); + typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff"); + typeColorMap.put(TEST_METHOD, "#91fae8"); + typeColorMap.put(PII, "#66ccff"); + typeColorMap.put(PURITY, "#ffe187"); + typeColorMap.put(IMAGE, "#fcc5fb"); + typeColorMap.put(OCR, "#fcc5fb"); + typeColorMap.put(LOGO, "#ffe187"); + typeColorMap.put(FORMULA, "#ffe187"); + typeColorMap.put(SIGNATURE, "#ffe187"); + typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6"); + typeColorMap.put(ROTATE_SIMPLE, "#66ccff"); + + hintTypeMap.put(VERTEBRATE, true); + hintTypeMap.put(ADDRESS, false); + hintTypeMap.put(AUTHOR, false); + hintTypeMap.put(SPONSOR, false); + hintTypeMap.put(NO_REDACTION_INDICATOR, true); + hintTypeMap.put(REDACTION_INDICATOR, true); + hintTypeMap.put(HINT_ONLY, true); + hintTypeMap.put(MUST_REDACT, true); + hintTypeMap.put(PUBLISHED_INFORMATION, true); + hintTypeMap.put(TEST_METHOD, true); + hintTypeMap.put(PII, false); + hintTypeMap.put(PURITY, false); + hintTypeMap.put(IMAGE, true); + hintTypeMap.put(OCR, true); + hintTypeMap.put(FORMULA, false); + hintTypeMap.put(LOGO, false); + hintTypeMap.put(SIGNATURE, false); + hintTypeMap.put(DOSSIER_REDACTIONS, false); + hintTypeMap.put(IMPORTED_REDACTION, false); + hintTypeMap.put(ROTATE_SIMPLE, false); + + caseInSensitiveMap.put(VERTEBRATE, true); + caseInSensitiveMap.put(ADDRESS, false); + caseInSensitiveMap.put(AUTHOR, false); + caseInSensitiveMap.put(SPONSOR, false); + caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); + caseInSensitiveMap.put(REDACTION_INDICATOR, true); + caseInSensitiveMap.put(HINT_ONLY, true); + caseInSensitiveMap.put(MUST_REDACT, true); + caseInSensitiveMap.put(PUBLISHED_INFORMATION, true); + caseInSensitiveMap.put(TEST_METHOD, false); + caseInSensitiveMap.put(PII, false); + caseInSensitiveMap.put(PURITY, false); + caseInSensitiveMap.put(IMAGE, true); + caseInSensitiveMap.put(OCR, true); + caseInSensitiveMap.put(SIGNATURE, true); + caseInSensitiveMap.put(LOGO, true); + caseInSensitiveMap.put(FORMULA, true); + caseInSensitiveMap.put(DOSSIER_REDACTIONS, false); + caseInSensitiveMap.put(IMPORTED_REDACTION, false); + caseInSensitiveMap.put(ROTATE_SIMPLE, true); + + recommendationTypeMap.put(VERTEBRATE, false); + recommendationTypeMap.put(ADDRESS, false); + recommendationTypeMap.put(AUTHOR, false); + recommendationTypeMap.put(SPONSOR, false); + recommendationTypeMap.put(NO_REDACTION_INDICATOR, false); + recommendationTypeMap.put(REDACTION_INDICATOR, false); + recommendationTypeMap.put(HINT_ONLY, false); + recommendationTypeMap.put(MUST_REDACT, false); + recommendationTypeMap.put(PUBLISHED_INFORMATION, false); + recommendationTypeMap.put(TEST_METHOD, false); + recommendationTypeMap.put(PII, false); + recommendationTypeMap.put(PURITY, false); + recommendationTypeMap.put(IMAGE, false); + recommendationTypeMap.put(OCR, false); + recommendationTypeMap.put(FORMULA, false); + recommendationTypeMap.put(SIGNATURE, false); + recommendationTypeMap.put(LOGO, false); + recommendationTypeMap.put(DOSSIER_REDACTIONS, false); + recommendationTypeMap.put(IMPORTED_REDACTION, false); + recommendationTypeMap.put(ROTATE_SIMPLE, false); + + rankTypeMap.put(PURITY, 155); + rankTypeMap.put(PII, 150); + rankTypeMap.put(ADDRESS, 140); + rankTypeMap.put(AUTHOR, 130); + rankTypeMap.put(SPONSOR, 120); + rankTypeMap.put(VERTEBRATE, 110); + rankTypeMap.put(MUST_REDACT, 100); + rankTypeMap.put(REDACTION_INDICATOR, 90); + rankTypeMap.put(NO_REDACTION_INDICATOR, 80); + rankTypeMap.put(PUBLISHED_INFORMATION, 70); + rankTypeMap.put(TEST_METHOD, 60); + rankTypeMap.put(HINT_ONLY, 50); + rankTypeMap.put(IMAGE, 30); + rankTypeMap.put(OCR, 29); + rankTypeMap.put(LOGO, 28); + rankTypeMap.put(SIGNATURE, 27); + rankTypeMap.put(FORMULA, 26); + rankTypeMap.put(DOSSIER_REDACTIONS, 200); + rankTypeMap.put(IMPORTED_REDACTION, 200); + rankTypeMap.put(ROTATE_SIMPLE, 150); + + colors.setSkippedColor("#cccccc"); + colors.setRequestAddColor("#04b093"); + colors.setRequestRemoveColor("#04b093"); + } + + + @SneakyThrows + private void loadNerForTest() { + + ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + } + + + private List getTypeResponse() { + + return typeColorMap.entrySet() + .stream() + .map(typeColor -> Type.builder() + .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(typeColor.getKey()) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .hexColor(typeColor.getValue()) + .isHint(hintTypeMap.get(typeColor.getKey())) + .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) + .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) + .rank(rankTypeMap.get(typeColor.getKey())) + .build()) + + .collect(Collectors.toList()); + } + + protected List getPathsRecursively(File path) { + + List result = new ArrayList<>(); + if (path == null || path.listFiles() == null) { + return result; + } + for (File f : path.listFiles()) { + if (f.isFile()) { + result.add(f); + } else { + result.addAll(getPathsRecursively(f)); + } + } + return result; + + } + + protected void loadOnlyDictionaryForSimpleFile() { + + dictionary.clear(); + dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + } + + + private void loadDictionaryForTest() { + + dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); + + falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(String file) { + + return prepareStorage(file, "files/cv_service_empty_response.json"); + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) { + + ClassPathResource pdfFileResource = new ClassPathResource(file); + ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile); + + return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream()); + } + + + @SneakyThrows + protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) { + + AnalyzeRequest request = AnalyzeRequest.builder() + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .lastProcessed(OffsetDateTime.now()) + .build(); + + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); + storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); + + return request; + } + + + @After + public void cleanupStorage() { + + if (this.storageService instanceof FileSystemBackedStorageService) { + ((FileSystemBackedStorageService) this.storageService).clearStorage(); + } + } + + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class}) + public static class TestConfiguration { + + @Bean + public KieContainer kieContainer() { + + KieServices kieServices = KieServices.Factory.get(); + + KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); + InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8)); + kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); + KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); + kieBuilder.buildAll(); + KieModule kieModule = kieBuilder.getKieModule(); + + return kieServices.newKieContainer(kieModule.getReleaseId()); + } + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(); + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java new file mode 100644 index 00000000..2db1dce0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -0,0 +1,65 @@ +package com.iqser.red.service.redaction.v1.server; + +import java.util.List; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity; +import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; + +import lombok.SneakyThrows; + +public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { + + @Autowired + private DocumentGraphFactory documentGraphFactory; + + @Autowired + private PdfSegmentationService segmentationService; + + @Autowired + private DictionaryService dictionaryService; + + + @Test + @SneakyThrows + public void testBuildDocumentEntity() { + + String filename = "files/new/crafted document"; + + prepareStorage(filename + ".pdf"); + ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + + var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); + DocumentEntity document = documentGraphFactory.buildEntityFromDocument(classifiedDoc); + + var start = System.currentTimeMillis(); + + TextBlockEntity textBlock = document.buildTextBlock(); + dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + + for (var model : dictionary.getDictionaryModels()) { + List entries = model.getEntriesSearch().getMatches(textBlock); + List falsePositives = model.getFalsePositiveSearch().getMatches(textBlock); + List falseRecommendations = model.getFalseRecommendationsSearch().getMatches(textBlock); + + if (!entries.isEmpty()) { + var pos = entries.get(0); + System.out.println(textBlock.getPositions(pos.startIndex(), pos.endIndex())); + } + + } + + System.out.printf("Search took %fs", ((float) (System.currentTimeMillis() - start)) / 1000); + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/RSS/07 - Acute Oral Toxicity in the Rat- Up and D.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/RSS/07 - Acute Oral Toxicity in the Rat- Up and D.pdf deleted file mode 100644 index 2caf5318..00000000 Binary files a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/RSS/07 - Acute Oral Toxicity in the Rat- Up and D.pdf and /dev/null differ