Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009: Document Tree Structure

Browse Source 
* added a TableFactory to refactor DocumentGraphFactory
* updated all enabled tests to run with new structure
* moved PdfTableCells to classification package
This commit is contained in:
Kilian Schuettler 2023-04-26 19:20:12 +02:00
parent 1f7caeebf6
commit 8682b01861
31 changed files with 1175 additions and 927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/TableServiceResponseAdapter.java
View File

@ -10,9 +10,9 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableCells;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableServiceResponse;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfTableCell.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/table/PdfTableCell.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
import lombok.AllArgsConstructor;
import lombok.Builder;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java
View File

@ -18,6 +18,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.TableServiceResponseAdapter;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
@ -27,7 +28,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.RequiredArgsConstructor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/RulingCleaningService.java
View File

@ -12,10 +12,10 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.RequiredArgsConstructor;

172
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
@ -20,10 +19,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
@ -35,29 +31,28 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentGraphFactory {
public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
public DocumentGraph buildDocumentGraph(Document document) {
TextBlockFactory textBlockFactory = new TextBlockFactory();
DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.getPages().put(page, new AtomicInteger(1)));
document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image));
addSections(document, context);
addHeaderAndFooterToEachPage(document, context);
@ -75,114 +70,21 @@ public class DocumentGraphFactory {
}
public void addTable(SemanticNode parentNode, TablePageBlock table, Context context) {
PageNode page = getPage(table.getPage(), context);
TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
if (!page.getMainBody().contains(parentNode)) {
parentNode.getPages().add(page);
}
page.getMainBody().add(tableNode);
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
tableNode.setTocId(tocId);
addTableCells(table.getRows(), tableNode, context, table.getPage());
}
private void addTableCells(List<List<Cell>> rows, TableNode tableNode, Context context, int pageNumber) {
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, pageNumber, context);
}
}
}
private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, Context context) {
PageNode page = getPage(pageNumber, context);
cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
TableCellNode tableCellNode = TableCellNode.builder()
.tableOfContents(context.tableOfContents())
.row(rowIndex)
.col(colIndex)
.header(cell.isHeaderCell())
.bBox(cell.getBounds2D())
.build();
page.getMainBody().add(tableCellNode);
com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock textBlock;
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
tableCellNode.setTocId(tocId);
if (cell.getTextBlocks().isEmpty()) {
tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page));
tableCellNode.setTerminal(true);
} else if (cell.getTextBlocks().size() == 1) {
textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
} else if (firstTextBlockIsHeadline(cell)) {
SectionNodeFactory.addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
} else {
cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context));
tableCellNode.setTerminal(false);
}
}
private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) {
return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
}
private static boolean firstTextBlockIsHeadline(Cell cell) {
String classification = cell.getTextBlocks().get(0).getClassification();
return classification != null && classification.startsWith("H");
}
public static boolean pageBlockIsHeadline(AbstractPageBlock abstractPageBlock) {
return abstractPageBlock instanceof ClassificationTextBlock && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().startsWith("H");
}
private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context) {
addParagraphOrHeadline(parentNode, originalTextBlock, context, emptyList());
}
public void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, List<ClassificationTextBlock> textBlocksToMerge) {
PageNode page = getPage(originalTextBlock.getPage(), context);
PageNode page = context.getPage(originalTextBlock.getPage());
SemanticNode node;
if (pageBlockIsHeadline(originalTextBlock)) {
node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
node = HeadlineNode.builder().tableOfContents(context.getTableOfContents()).build();
} else {
node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
node = ParagraphNode.builder().tableOfContents(context.getTableOfContents()).build();
}
page.getMainBody().add(node);
@ -207,18 +109,18 @@ public class DocumentGraphFactory {
public void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
PageNode page = getPage(image.getPage(), context);
PageNode page = context.getPage(image.getPage());
ImageNode imageNode = ImageNode.builder()
.id(IdBuilder.buildId(Set.of(page), List.of(position)))
.imageType(image.getImageType())
.position(position)
.transparent(image.isHasTransparency())
.page(page)
.tableOfContents(context.tableOfContents())
.tableOfContents(context.getTableOfContents())
.build();
page.getMainBody().add(imageNode);
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(sectionNode.getTocId(), NodeType.IMAGE, imageNode);
List<Integer> tocId = context.getTableOfContents().createNewChildEntryAndReturnId(sectionNode.getTocId(), NodeType.IMAGE, imageNode);
imageNode.setTocId(tocId);
}
@ -257,13 +159,13 @@ public class DocumentGraphFactory {
private void addFooter(List<ClassificationTextBlock> textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
PageNode page = context.getPage(textBlocks.get(0).getPage());
FooterNode footer = FooterNode.builder().tableOfContents(context.getTableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
footer,
context,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
List<Integer> tocId = context.getTableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock);
page.setFooter(footer);
@ -272,14 +174,14 @@ public class DocumentGraphFactory {
public void addHeader(List<ClassificationTextBlock> textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
PageNode page = context.getPage(textBlocks.get(0).getPage());
HeaderNode header = HeaderNode.builder().tableOfContents(context.getTableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
header,
context,
0,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
List<Integer> tocId = context.getTableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId);
header.setTerminalTextBlock(textBlock);
page.setHeader(header);
@ -288,10 +190,10 @@ public class DocumentGraphFactory {
private void addEmptyFooter(int pageIndex, Context context) {
PageNode page = getPage(pageIndex, context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
PageNode page = context.getPage(pageIndex);
FooterNode footer = FooterNode.builder().tableOfContents(context.getTableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
List<Integer> tocId = context.getTableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock);
page.setFooter(footer);
@ -300,10 +202,10 @@ public class DocumentGraphFactory {
private void addEmptyHeader(int pageIndex, Context context) {
PageNode page = getPage(pageIndex, context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
PageNode page = context.getPage(pageIndex);
HeaderNode header = HeaderNode.builder().tableOfContents(context.getTableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
List<Integer> tocId = context.getTableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId);
header.setTerminalTextBlock(textBlock);
page.setHeader(header);
@ -322,18 +224,26 @@ public class DocumentGraphFactory {
}
public PageNode getPage(int pageIndex, Context context) {
@Getter
@Builder
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
static final class Context {
return context.pages.keySet()
.stream()
.filter(page -> page.getNumber() == pageIndex)
.findFirst()
.orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
}
TableOfContents tableOfContents;
Map<PageNode, AtomicInteger> pages;
List<SectionNode> sections;
List<ClassifiedImage> images;
TextBlockFactory textBlockFactory;
record Context(
TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<ClassifiedImage> images, TextBlockFactory textBlockFactory) {
public PageNode getPage(int pageIndex) {
return pages.keySet()
.stream()
.filter(page -> page.getNumber() == pageIndex)
.findFirst()
.orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
}
}

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java
View File

@ -59,17 +59,20 @@ public class SearchTextWithTextPositionFactory {
lastHyphenIdx = stringIdx;
}
sb.append(currentTextPosition.getUnicode());
stringIdxToPositionIdx.add(positionIdx);
++stringIdx;
// unicode characters with more than 16 bit encoding have a length > 1 in java strings
for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) {
stringIdxToPositionIdx.add(positionIdx);
}
stringIdx += currentTextPosition.getUnicode().length();
}
previousTextPosition = currentTextPosition;
++positionIdx;
}
previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build();
sb.append(previousTextPosition.getUnicode());
sb.append(" ");
stringIdxToPositionIdx.add(positionIdx);
++stringIdx;
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
View File

@ -30,16 +30,16 @@ public class SectionNodeFactory {
return;
}
Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.getTableOfContents()).build();
context.sections().add(sectionNode);
context.getSections().add(sectionNode);
blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
List<Integer> tocId;
if (parentNode == null) {
tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
tocId = context.getTableOfContents().createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
} else {
tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
tocId = context.getTableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
}
sectionNode.setTocId(tocId);
@ -89,7 +89,7 @@ public class SectionNodeFactory {
DocumentGraphFactory.addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractPageBlock, context, textBlocks);
}
if (abstractPageBlock instanceof TablePageBlock) {
DocumentGraphFactory.addTable(sectionNode, (TablePageBlock) abstractPageBlock, context);
TableNodeFactory.addTable(sectionNode, (TablePageBlock) abstractPageBlock, context);
}
}
}
@ -180,7 +180,7 @@ public class SectionNodeFactory {
private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, SectionNode sectionNode, Integer pageNumber) {
PageNode page = DocumentGraphFactory.getPage(pageNumber, context);
PageNode page = context.getPage(pageNumber);
page.getMainBody().add(sectionNode);
}

122
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java Normal file
View File

@ -0,0 +1,122 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import static java.util.Collections.emptyList;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
import lombok.experimental.UtilityClass;
@UtilityClass
public class TableNodeFactory {
public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
public void addTable(SemanticNode parentNode, TablePageBlock table, DocumentGraphFactory.Context context) {
PageNode page = context.getPage(table.getPage());
TableNode tableNode = TableNode.builder().tableOfContents(context.getTableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
if (!page.getMainBody().contains(parentNode)) {
parentNode.getPages().add(page);
}
page.getMainBody().add(tableNode);
List<Integer> tocId = context.getTableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
tableNode.setTocId(tocId);
addTableCells(table.getRows(), tableNode, context, table.getPage());
IfTableHasNoHeadersAssumeFirstRowAreHeaders(tableNode);
}
private static void IfTableHasNoHeadersAssumeFirstRowAreHeaders(TableNode tableNode) {
if (tableNode.streamHeaders().findAny().isEmpty()) {
tableNode.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true));
}
}
private void addTableCells(List<List<Cell>> rows, TableNode tableNode, DocumentGraphFactory.Context context, int pageNumber) {
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, pageNumber, context);
}
}
}
private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, DocumentGraphFactory.Context context) {
PageNode page = context.getPage(pageNumber);
cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
TableCellNode tableCellNode = TableCellNode.builder()
.tableOfContents(context.getTableOfContents())
.row(rowIndex)
.col(colIndex)
.header(cell.isHeaderCell())
.bBox(cell.getBounds2D())
.build();
page.getMainBody().add(tableCellNode);
com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock textBlock;
List<Integer> tocId = context.getTableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
tableCellNode.setTocId(tocId);
if (cell.getTextBlocks().isEmpty()) {
tableCellNode.setTerminalTextBlock(context.getTextBlockFactory().emptyTextBlock(parentNode, context, page));
tableCellNode.setTerminal(true);
} else if (cell.getTextBlocks().size() == 1) {
textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
} else if (firstTextBlockIsHeadline(cell)) {
SectionNodeFactory.addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
} else {
cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCellNode, tb, context, emptyList()));
tableCellNode.setTerminal(false);
}
}
private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) {
return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
}
private static boolean firstTextBlockIsHeadline(Cell cell) {
String classification = cell.getTextBlocks().get(0).getClassification();
return classification != null && classification.startsWith("H");
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java
View File

@ -26,7 +26,7 @@ public class TextBlockFactory {
public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
Integer numberOnPage = context.pages().get(page).getAndIncrement();
Integer numberOnPage = context.getPages().get(page).getAndIncrement();
return buildAtomicTextBlock(sequences, parent, context, numberOnPage, page);
}
@ -56,7 +56,7 @@ public class TextBlockFactory {
public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page);
return emptyTextBlock(parent, context.getPages().get(page).getAndIncrement(), page);
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
View File

@ -4,6 +4,7 @@ import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -46,7 +47,7 @@ public class RedactionEntity {
Set<Engine> engines;
Set<RedactionEntity> references;
@Builder.Default
List<Integer> matchedRules = new LinkedList<>();
Deque<Integer> matchedRules = new LinkedList<>();
String redactionReason;
String legalBasis;
@ -131,7 +132,7 @@ public class RedactionEntity {
if (matchedRules.isEmpty()) {
return -1;
}
return matchedRules.get(matchedRules.size() - 1);
return matchedRules.getLast();
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/DocumentGraph.java
View File

@ -86,6 +86,12 @@ public class DocumentGraph implements SemanticNode {
}
public Stream<ImageNode> streamAllImages() {
return streamAllSubNodes().filter(node -> node instanceof ImageNode).map(node -> (ImageNode) node);
}
@Override
public String toString() {

55
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -14,6 +14,7 @@ import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
@ -41,7 +42,7 @@ public class EntityCreationService {
private final EntityEnrichmentService entityEnrichmentService;
public Set<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
List<Boundary> startBoundaries = RedactionSearchUtils.findBoundariesByString(start, textBlock);
@ -49,7 +50,7 @@ public class EntityCreationService {
List<Boundary> entityBoundaries = new LinkedList<>();
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
return Collections.emptySet();
return Stream.empty();
}
for (Boundary startBoundary : startBoundaries) {
Optional<Boundary> optionalFirstStopBoundary = stopBoundaries.stream().filter(stopBoundary -> stopBoundary.end() > startBoundary.start()).findFirst();
@ -59,24 +60,20 @@ public class EntityCreationService {
stopBoundaries.remove(optionalFirstStopBoundary.get());
entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
}
return entityBoundaries.stream()
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
return entityBoundaries.stream().filter(boundary -> isValidEntityBoundary(textBlock, boundary)).map(boundary -> byBoundary(boundary, type, entityType, node));
}
public Set<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary))
.map(bounds -> byBoundary(bounds, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
.map(bounds -> byBoundary(bounds, type, entityType, node));
}
public Set<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, false);
@ -84,42 +81,40 @@ public class EntityCreationService {
.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
.map(boundary -> byBoundary(boundary, type, entityType, node));
}
public Set<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByString(string, textBlock);
return boundaries.stream()
return RedactionSearchUtils.findBoundariesByString(string, textBlock)
.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
.map(boundary -> byBoundary(boundary, type, entityType, node));
}
public Set<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
return RedactionSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
}
public Set<RedactionEntity> byKeyword(String keyword, String type, EntityType entityType, SemanticNode node) {
public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByString(keyword, node.buildTextBlock());
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
return RedactionSearchUtils.findBoundariesByString(keyword, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
}
public RedactionEntity bySemanticNode(SemanticNode node, String type, EntityType entityType) {
Boundary boundary = node.buildTextBlock().getBoundary();
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
return byBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
if (boundary.length() > 0) {
boundary = new Boundary(boundary.start(), boundary.end() - 1);
}
return byBoundary(boundary, type, entityType, node);
}
@ -133,10 +128,20 @@ public class EntityCreationService {
public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
int expandedEnd = getExpandedEndByRegex(entity, regexPattern);
expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().buildTextBlock());
return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
}
private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
if (textBlock.getNextLinebreak(end) < expandedEnd) {
return end;
}
return expandedEnd;
}
public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/conditions/SemanticNodeRuleConditions.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
package com.iqser.red.service.redaction.v1.server.redaction.conditions;
import java.util.List;

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java
View File

@ -1,19 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.SearchTextWithTextPositionDto;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter
@Builder
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class Paragraph {
SearchTextWithTextPositionDto searchTextToTextPosition;
int sectionNumber;
int paragraphNumber;
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfTable.java
View File

@ -1,17 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@RequiredArgsConstructor
public class PdfTable {
@NonNull
private List<PdfTableCell> tableCells = new ArrayList<>();
}

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java
View File

@ -1,16 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class SectionSearchableTextPair {
private Section section;
private SearchableText searchableText;
private List<Integer> cellStarts;
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -25,7 +25,6 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@ -69,14 +68,14 @@ public class DroolsExecutionService {
ManualRedactions manualRedactions,
List<EntityRecognitionEntity> nerEntities) {
return executeRules(kieContainer, document, document.getMainSections(), dictionary, fileAttributes, manualRedactions, nerEntities);
return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
}
@Timed("redactmanager_executeRules")
public List<FileAttribute> executeRules(KieContainer kieContainer,
DocumentGraph document,
List<SectionNode> sectionsToAnalyze,
List<SemanticNode> sectionsToAnalyze,
Dictionary dictionary,
List<FileAttribute> fileAttributes,
ManualRedactions manualRedactions,

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -33,15 +33,21 @@ public class RedactionLogCreatorService {
List<RedactionLogEntry> entries = new ArrayList<>();
Set<String> processedIds = new HashSet<>();
documentGraph.getEntities().forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
documentGraph.streamAllSubNodes()
.filter(node -> node instanceof ImageNode)
.map(node -> (ImageNode) node)
.forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
documentGraph.getEntities()
.stream()
.filter(RedactionLogCreatorService::isEntityOrRecommendationType)
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
documentGraph.streamAllImages().forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
return entries;
}
private static boolean isEntityOrRecommendationType(RedactionEntity redactionEntity) {
return redactionEntity.getEntityType() == EntityType.ENTITY || redactionEntity.getEntityType() == EntityType.RECOMMENDATION;
}
private List<RedactionLogEntry> toRedactionLogEntries(RedactionEntity redactionEntity, Set<String> processedIds, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -31,13 +31,13 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedSectionText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.Text;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
@ -117,8 +117,6 @@ public class AnalyzeService {
sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount, sectionTexts);
Text text = new Text(pageCount, sectionTexts);
log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(documentGraph));
@ -128,7 +126,7 @@ public class AnalyzeService {
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(System.currentTimeMillis() - startTime)
.numberOfPages(text.getNumberOfPages())
.numberOfPages(documentGraph.getNumberOfPages())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.build();
}
@ -161,18 +159,18 @@ public class AnalyzeService {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
}
List<SectionNode> sectionsToReAnalyse = documentGraph.getMainSections()
.stream()
List<SemanticNode> sectionsToReAnalyse = documentGraph.streamChildren()
.filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0)))
.collect(Collectors.toList());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode));
List<EntityRecognitionEntity> nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph);
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary,
Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary,
documentGraph,
sectionsToReAnalyse,
kieContainer,
@ -205,15 +203,8 @@ public class AnalyzeService {
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary,
documentGraph,
kieContainer,
analyzeRequest,
nerEntities);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -20,7 +20,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
@ -59,14 +58,12 @@ class SectionFinder {
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
true);
for (SectionNode section : documentGraph.getMainSections()) {
if (dictionaryIncrementsSearch.atLeastOneMatches(section.buildTextBlock().getSearchText())) {
sectionsToReanalyse.add(section.getTocId().get(0));
documentGraph.streamChildren().forEach(mainNode -> {
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.buildTextBlock().getSearchText())) {
sectionsToReanalyse.add(mainNode.getTocId().get(0));
}
}
});
log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
return sectionsToReanalyse;

63
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -10,10 +9,11 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
@ -36,11 +36,16 @@ public class EntityRedactionService {
EntityCreationService entityCreationService;
public Set<FileAttribute> addRuleEntitiesToGraphAndReturnAddedFileAttributes(Dictionary dictionary,
DocumentGraph documentGraph,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
addDictionaryEntities(dictionary, documentGraph);
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
log.debug("Starting Drools Execution");
long droolsStart = System.currentTimeMillis();
@ -55,13 +60,20 @@ public class EntityRedactionService {
}
public Set<FileAttribute> addRuleEntitiesToGraphAndReturnAddedFileAttributes(Dictionary dictionary,
DocumentGraph documentGraph,
List<SectionNode> sectionsToReanalyze,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
List<SemanticNode> sectionsToReanalyze,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
List<EntityRecognitionEntity> nerEntities) {
log.debug("Starting Dictionary Search");
long dictSearchStart = System.currentTimeMillis();
sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node));
log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
log.debug("Starting Drools execution");
long ruleExecutionStart = System.currentTimeMillis();
List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
documentGraph,
sectionsToReanalyze,
@ -69,34 +81,39 @@ public class EntityRedactionService {
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.debug("Finished Drools execution in {} ms", System.currentTimeMillis() - ruleExecutionStart);
return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
}
public void addDictionaryEntities(Dictionary dictionary, DocumentGraph document) {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
List<RedactionEntity> foundEntities = new LinkedList<>();
for (var model : dictionary.getDictionaryModels()) {
findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
findEntitiesWithSearchImplementation(node, model.getEntriesSearch(), EntityType.ENTITY, model.isDossierDictionary(), model.getType());
findEntitiesWithSearchImplementation(node, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, model.isDossierDictionary(), model.getType());
findEntitiesWithSearchImplementation(node, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, model.isDossierDictionary(), model.getType());
}
foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
}
private void findEntitiesWithSearchImplementation(DocumentGraph documentGraph,
private void findEntitiesWithSearchImplementation(SemanticNode node,
SearchImplementation searchImplementation,
EntityType entityType,
List<RedactionEntity> foundEntities,
boolean isDossierDictionary,
String type) {
TextBlock textBlock = documentGraph.getTextBlock();
TextBlock textBlock = node.buildTextBlock();
searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
.stream()
.filter(boundary -> entityCreationService.isValidEntityBoundary(textBlock, boundary))
.map(bounds -> RedactionEntity.initialEntityNode(bounds, type, entityType))
.forEach(foundEntities::add);
.forEach(entity -> {
entity.setDossierDictionaryEntry(isDossierDictionary);
entity.setDictionaryEntry(true);
entity.addEngine(Engine.DICTIONARY);
entityCreationService.addEntityToGraph(entity, node);
});
}
}

240
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java
View File

@ -1,41 +1,21 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.mockito.stubbing.Answer;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
@ -45,26 +25,16 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
@ -80,57 +50,18 @@ import lombok.ToString;
@Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
public class HeadlinesGoldStandardIntegrationTest {
private static final String RULES = loadFromClassPath("drools/headlines.drl");
private static final String HEADLINE = "headline";
@Autowired
private RedactionController redactionController;
@Autowired
private AnnotationService annotationService;
@Autowired
private AnalyzeService analyzeService;
@Autowired
private ObjectMapper objectMapper;
@MockBean
private RulesClient rulesClient;
@MockBean
private DictionaryClient dictionaryClient;
@Autowired
private RedactionStorageService redactionStorageService;
@Autowired
private StorageService storageService;
@Autowired
private ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
@MockBean
private AmazonS3 amazonS3;
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private LegalBasisClient legalBasisClient;
private final Map<String, List<String>> dictionary = new HashMap<>();
private final Map<String, List<String>> dossierDictionary = new HashMap<>();
private final Map<String, String> typeColorMap = new HashMap<>();
private final Map<String, Boolean> hintTypeMap = new HashMap<>();
private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
private final Map<String, Integer> rankTypeMap = new HashMap<>();
private final Colors colors = new Colors();
private final Map<String, Long> reanlysisVersions = new HashMap<>();
private final Set<String> deleted = new HashSet<>();
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
private final static String TEST_DOSSIER_ID = "123";
private final static String TEST_FILE_ID = "123";
@ -146,15 +77,8 @@ public class HeadlinesGoldStandardIntegrationTest {
"files/Headlines/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
metrics.add(getMetrics("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/Headlines/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
float precision = 0;
float recall = 0;
for (var m : metrics) {
precision += m.getPrecision();
recall += m.getRecall();
}
precision = precision / metrics.size();
recall = recall / metrics.size();
double precision = metrics.stream().mapToDouble(Metrics::getPrecision).average().orElse(1.0);
double recall = metrics.stream().mapToDouble(Metrics::getRecall).average().orElse(1.0);
System.out.println("Precision is: " + precision + " recall is: " + recall);
@ -175,11 +99,13 @@ public class HeadlinesGoldStandardIntegrationTest {
AnalyzeRequest request = prepareStorage(fileUrl);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
List<Headline> foundHeadlines = new ArrayList<>();
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(e -> foundHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
var foundHeadlines = documentGraph.streamAllSubNodes()
.map(SemanticNode::getHeadline)
.distinct()
.map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.buildTextBlock().getSearchText().stripTrailing()))
.toList();
Set<Headline> correct = new HashSet<>();
Set<Headline> missing;
@ -194,11 +120,10 @@ public class HeadlinesGoldStandardIntegrationTest {
missing = goldStandardHeadlines.stream().filter(h -> !correct.contains(h)).collect(Collectors.toSet());
float precision = (float) correct.size() / ((float) correct.size() + (float) falsePositive.size());
float precision = (float) correct.size() / (float) foundHeadlines.size();
float recall = (float) correct.size() / ((float) correct.size() + (float) missing.size());
return new Metrics(precision, recall);
}
@ -207,22 +132,6 @@ public class HeadlinesGoldStandardIntegrationTest {
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/test/resources/drools/headlines.drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
@Bean
@Primary
public StorageService inmemoryStorage() {
@ -242,104 +151,6 @@ public class HeadlinesGoldStandardIntegrationTest {
}
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
mockDictionaryCalls(null);
mockDictionaryCalls(0L);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
private void mockDictionaryCalls(Long version) {
when(dictionaryClient.getDictionaryForType(HEADLINE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HEADLINE, false));
}
private void loadDictionaryForTest() {
dictionary.computeIfAbsent(HEADLINE, v -> new ArrayList<>());
}
private void loadTypeForTest() {
typeColorMap.put(HEADLINE, "#f90707");
hintTypeMap.put(HEADLINE, false);
caseInSensitiveMap.put(HEADLINE, false);
recommendationTypeMap.put(HEADLINE, false);
rankTypeMap.put(HEADLINE, 155);
colors.setSkippedColor("#cccccc");
colors.setRequestAddColor("#04b093");
colors.setRequestRemoveColor("#04b093");
}
private List<Type> getTypeResponse() {
return typeColorMap.entrySet()
.stream()
.map(typeColor -> Type.builder()
.id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(typeColor.getKey())
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColor.getValue())
.isHint(hintTypeMap.get(typeColor.getKey()))
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
.rank(rankTypeMap.get(typeColor.getKey()))
.build())
.collect(Collectors.toList());
}
private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
return Type.builder()
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColorMap.get(type))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
.falsePositiveEntries(new ArrayList<>())
.falseRecommendationEntries(new ArrayList<>())
.isHint(hintTypeMap.get(type))
.isCaseInsensitive(caseInSensitiveMap.get(type))
.isRecommendation(recommendationTypeMap.get(type))
.rank(rankTypeMap.get(type))
.build();
}
private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
if (entries == null) {
entries = Collections.emptyList();
}
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
entries.forEach(entry -> {
dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
});
return dictionaryEntries;
}
@SneakyThrows
private AnalyzeRequest prepareStorage(String file) {
@ -376,35 +187,6 @@ public class HeadlinesGoldStandardIntegrationTest {
}
private static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
if (resource == null) {
throw new IllegalArgumentException("could not load classpath resource: drools/prod_rules_new.drl");
}
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
StringBuilder sb = new StringBuilder();
String str;
while ((str = br.readLine()) != null) {
sb.append(str).append("\n");
}
return sb.toString();
} catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
}
}
@SneakyThrows
private void loadNerForTest() {
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream());
}
@Data
@EqualsAndHashCode
@AllArgsConstructor

82
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,15 +1,13 @@
package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -27,11 +25,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
@ -66,7 +59,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -87,22 +82,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class RedactionIntegrationTestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
@Bean
@Primary
public StorageService inmemoryStorage() {
@ -207,7 +186,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
public void redactionExpansionOverlap() throws IOException {
public void redactionExpansionOverlap() {
// F. Lastname, J. Doe, M. Mustermann
// Lastname M., Doe J., Mustermann M.
@ -347,6 +326,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
@Disabled // TODO: this is already broken on master, no idea how to fix it. Most likely more responses need to be stubbed.
public void redactionTestSeparatedRedaction() throws IOException {
String fileName = "scanned/VV-380943_page38.pdf";
@ -367,7 +347,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
long end = System.currentTimeMillis();
@ -380,13 +360,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
int correctFound = 0;
loop:
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
for (SectionText sectionText : text.getSectionTexts()) {
for (SectionNode section : documentGraph.getMainSections()) {
if (redactionLogEntry.isImage()) {
correctFound++;
continue loop;
}
if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
if (redactionLogEntry.getSectionNumber() == section.getTocId().get(0)) {
String value = section.buildTextBlock().subSequence(new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString();
if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
correctFound++;
} else {
@ -521,7 +501,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
long end = System.currentTimeMillis();
@ -531,25 +511,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
}
int correctFound = 0;
loop:
for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
for (SectionText sectionText : text.getSectionTexts()) {
if (redactionLogEntry.isImage()) {
correctFound++;
continue loop;
}
if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
correctFound++;
} else {
throw new RuntimeException("WTF");
}
}
}
}
assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
List<String> valuesInDocument = redactionLog.getRedactionLogEntry()
.stream()
.filter(e -> !e.isImage())
.map(redactionLogEntry -> new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
.map(boundary -> documentGraph.buildTextBlock().subSequence(boundary).toString())
.toList();
List<String> valuesInRedactionLog = redactionLog.getRedactionLogEntry().stream().filter(e -> !e.isImage()).map(RedactionLogEntry::getValue).toList();
assertEquals(valuesInRedactionLog, valuesInDocument);
dictionary.get(DICTIONARY_AUTHOR).add("properties");
reanlysisVersions.put("properties", 1L);
@ -974,7 +944,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
redactionLog.getRedactionLogEntry().forEach(entry -> {
if (!entry.isHint()) {
assertThat(entry.getReason()).isEqualTo("Not redacted because row is not a vertebrate study");
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
}
});
}
@ -1185,7 +1155,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
public void testImportedRedactions() throws IOException {
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
String outputFileName = OsUtils.getTemporaryDirectory() + "/ImportedRedactions.pdf";
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
@ -1200,6 +1170,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.getValue() == null) {
return;
@ -1211,10 +1185,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertThat(entry.getImportedRedactionIntersections()).isEmpty();
}
});
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
@ -150,6 +151,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
String entryAuthorDictionary = "Evans P.G.";
dictionary.put(DICTIONARY_AUTHOR, List.of(entryAuthorDictionary));
falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:"));
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationIntegrationTest.java
View File

@ -36,7 +36,7 @@ public class DocumentGraphVisualizationIntegrationTest extends BuildDocumentGrap
@Disabled
public void visualizeMetolachlor2() {
String filename = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06";
String filename = "files/Fludioxonil/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21";
DocumentGraph documentGraph = buildGraph(filename);
TextBlock textBlock = documentGraph.buildTextBlock();
@ -50,7 +50,7 @@ public class DocumentGraphVisualizationIntegrationTest extends BuildDocumentGrap
@Disabled
public void visualizeRotatedTestDocument() {
String filename = "files/new/RotateTestFileWithImages";
String filename = "files/new/test1S1T1";
DocumentGraph documentGraph = buildGraph(filename);
TextBlock textBlock = documentGraph.buildTextBlock();

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java
View File

@ -10,6 +10,7 @@ import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.kie.api.KieServices;
@ -78,8 +79,9 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphInte
public void manualResizeRedactionTest() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> biggerEntities = entityCreationService.byKeyword("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph).collect(Collectors.toUnmodifiableSet());
Set<RedactionEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, documentGraph)
.collect(Collectors.toUnmodifiableSet());
RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
RedactionEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
@ -114,7 +116,7 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphInte
public void manualForceRedactionTest() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph).collect(Collectors.toUnmodifiableSet());
RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
@ -147,7 +149,7 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphInte
public void manualIDRemovalTest() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph).collect(Collectors.toUnmodifiableSet());
RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
@ -174,7 +176,7 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphInte
public void manualIDRemovalButAlsoForceRedactionTest() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph).collect(Collectors.toUnmodifiableSet());
RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
@ -207,7 +209,7 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphInte
public void manualIDRemovalNotApprovedTest() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
Set<RedactionEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph).collect(Collectors.toUnmodifiableSet());
RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -124,7 +124,7 @@ public class LiveDataIntegrationTest {
when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "prod_rules_new.drl").getInputStream());
var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream());
when(rulesClient.getRules(any())).thenReturn(JSONPrimitive.of(rules));
ObjectMapper objectMapper = new ObjectMapper();

0
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl
View File

659
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl Normal file
View File

@ -0,0 +1,659 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
import java.util.stream.Collectors
import java.util.Collection
import java.util.stream.Stream
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
global DocumentGraph document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
rule "add NER Entities of type CBI_author or CBI_address"
salience 999
when
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
then
RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
redactionEntity.addEngine(Engine.NER);
insert(redactionEntity);
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "0: Expand CBI_author entities with firstname initials"
no-loop true
when
$entityToExpand: RedactionEntity(type == "CBI_author",
value.matches("[^\\s]+"),
textAfter.startsWith(" "),
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
)
then
RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
expandedEntity.addMatchedRule(0);
$entityToExpand.removeFromGraph();
retract($entityToExpand);
insert(expandedEntity);
end
rule "0: Expand CBI_author and PII entities with salutation prefix"
when
$entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
expandedEntity.addMatchedRule(0);
insert(expandedEntity);
end
rule "1: Redacted because Section contains Vertebrate"
when
$section: SectionNode(hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(1);
redactionEntity.setRedactionReason("Vertebrate Found in this section");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
end
rule "2: Not Redacted because Section contains no Vertebrate"
when
$section: SectionNode(!hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(2);
redactionEntity.setRedactionReason("No Vertebrate Found in this section");
});
end
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
$section: SectionNode(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(3);
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator found in this section");
});
end
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
$section: SectionNode(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(4);
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator, but also redaction-indicator, found in this section");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
end
rule "5: Do not redact Names and Addresses if published information found"
when
$section: SectionNode(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
redactionEntity.addReferences(publishedInformationEntities);
});
end
rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
when
$table: TableNode(hasHeader("Author(s)"))
then
$table.streamTableCellsWithHeader("Author(s)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> insert(redactionEntity));
end
rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
when
$table: TableNode(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.setRedaction(false);
authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study");
authorEntity.setLegalBasis("");
authorEntity.addMatchedRule(6);
});
end
rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
when
$table: TableNode(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
authorEntity.addMatchedRule(7);
});
end
rule "8: Redact if must_redact entity is found"
when
$section: SectionNode(hasEntitiesOfType("must_redact"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("must_redact entry was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(8);
});
end
rule "9: Redact CBI_sponsor entities if preceded by \" batches produced at\""
when
$sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.setRedaction(true);
$sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company");
$sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
$sponsorEntity.addMatchedRule(9);
end
rule "10: Redact row if row contains \"determination of residues\" and livestock keyword"
when
$keyword: String() from List.of("livestock",
"live stock",
"tissue",
"tissues",
"liver",
"muscle",
"bovine",
"ruminant",
"ruminants")
$residueKeyword: String() from List.of("determination of residues", "determination of total residues")
$table: TableNode(containsStringIgnoreCase($residueKeyword)
&& containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
.forEach(keywordEntity -> insert(keywordEntity));
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(10);
});
end
rule "11: Redact if CTL/* or BL/* was found"
when
$section: SectionNode(excludesTables, (containsString("CTL/") || containsString("BL/")))
then
entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(11);
});
end
rule "12: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.addMatchedRule(12);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
rule "13: Add recommendation for Addresses in Test Organism sections"
when
$section: SectionNode(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(redactionEntity -> {
redactionEntity.setRedactionReason("Line after \"Source\" in Test Organism Section");
redactionEntity.addMatchedRule(13);
insert(redactionEntity);
});
end
rule "14: Add recommendation for Addresses in Test Animals sections"
when
$section: SectionNode(excludesTables, containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(redactionEntity -> {
redactionEntity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
redactionEntity.addMatchedRule(14);
insert(redactionEntity);
});
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "15: Redact all PII"
when
$pii: RedactionEntity(type == "PII", redaction == false)
then
$pii.setRedaction(true);
$pii.setRedactionReason("PII found");
$pii.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
$pii.addMatchedRule(15);
end
rule "16: Redact Emails by RegEx (Non vertebrate study)"
when
$section: SectionNode(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section)
.forEach(emailEntity -> {
emailEntity.setRedaction(true);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
emailEntity.addMatchedRule(16);
insert(emailEntity);
});
end
rule "17: Redact line after contact information keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(excludesTables, containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(17);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
});
end
rule "18: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(18);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
});
end
rule "19: Redact AUTHOR(S)"
when
FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true")
$section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(19);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "20: Redact PERFORMING LABORATORY"
when
$section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(20);
authorEntity.setRedactionReason("PERFORMING LABORATORY was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "21: Redact On behalf of Sequani Ltd.:"
when
$section: SectionNode(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(21);
authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "22: Redact On behalf of Syngenta Ltd.:"
when
$section: SectionNode(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(21);
authorEntity.setRedactionReason("On behalf of Syngenta Ltd.: Name Title was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "26: Redact signatures"
when
$signature: ImageNode(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(26);
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "27: Redact formulas"
when
$formula: ImageNode(imageType == ImageType.FORMULA)
then
$formula.setRedaction(true);
$formula.setMatchedRule(27);
$formula.setRedactionReason("Formula Found");
$formula.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "28: Redact logos"
when
$logo: ImageNode(imageType == ImageType.LOGO)
then
$logo.setRedaction(true);
$logo.setMatchedRule(28);
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "29: Redact Dossier Redactions"
when
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
then
$dossierRedaction.setRedaction(true);
$dossierRedaction.addMatchedRule(29);
$dossierRedaction.setRedactionReason("Dossier Redaction found");
$dossierRedaction.setLegalBasis("Article 39(1)(2) of Regulation (EC) No 178/2002");
end
rule "30: Remove Dossier redactions if file is confidential"
when
FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
then
$dossierRedaction.removeFromGraph();
retract($dossierRedaction)
end
rule "101: Redact CAS Number"
when
$table: TableNode(hasHeader("Sample #"))
then
$table.streamTableCellsWithHeader("Sample #")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(101);
redactionEntity.setRedactionReason("Sample # found in Header");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
insert(redactionEntity);
});
end
rule "102: Guidelines FileAttributes"
when
$section: SectionNode(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
then
RedactionSearchUtils.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
.map(boundary -> $section.buildTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
end
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
rule "Apply manual resize redaction"
salience 128
when
$resizeRedactions: ManualResizeRedaction($id: annotationId)
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
then
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions);
retract($resizeRedactions);
update($entityToBeResized);
end
rule "Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.removeFromGraph();
retract($entityToBeRemoved);
end
rule "Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: ImageNode($id == id)
then
$entityToBeRemoved.setIgnored(true);
end
rule "Apply force redaction"
salience 128
when
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.setLegalBasis($legalBasis);
$entityToForce.setRedaction(true);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
end
rule "Apply image recategorization"
salience 128
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: ImageNode($id == id)
then
$image.setImageType(parseImageType($imageType));
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
$first.removeFromGraph();
$second.removeFromGraph();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
retract($first);
retract($second);
insert(mergedEntity);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
$entity.removeFromGraph();
retract($entity)
end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 64
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 32
when
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
then
$lowerRank.removeFromGraph();
retract($lowerRank);
end
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
rule "remove duplicate FileAttributes"
salience 64
when
$first: FileAttribute($label: label, $value: value)
$second: FileAttribute(this != $first, label == $label, value == $value)
then
retract($second);
end
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
end

402
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -1,341 +1,169 @@
package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
global Section section
import java.util.List;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
import java.util.stream.Collectors
import java.util.Collection
import java.util.stream.Stream
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
// --------------------------------------- AI rules -------------------------------------------------------------------
global DocumentGraph document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global Dictionary dictionary
rule "0: Add CBI_author from ai"
when
Section(aiMatchesType("CBI_author"))
then
section.addAiEntities("CBI_author", "CBI_author");
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
when
Section(aiMatchesType("ORG"))
then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
end
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
rule "add NER Entities of type CBI_author or CBI_address"
salience 999
when
Section(aiMatchesType("STREET"))
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
redactionEntity.addEngine(Engine.NER);
insert(redactionEntity);
end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
when
Section(aiMatchesType("CITY"))
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
end
/* Syngenta specific laboratory recommendation */
rule "0: Recommend CTL/BL laboratory that start with BL or CTL"
when
Section(searchText.contains("CT") || searchText.contains("BL"))
then
/* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address");
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
rule "Always redact CBI_author"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
$cbiAuthor: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$cbiAuthor.addMatchedRule(0);
$cbiAuthor.setRedaction(true);
$cbiAuthor.setRedactionReason("Author found");
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "2: Redact CBI Authors (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "3: Redact not CBI Address (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
end
rule "4: Redact CBI Address (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "5: Do not redact genitive CBI_author"
when
Section(matchesType("CBI_author"))
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
then
section.recommendLineAfter("Source", "CBI_address");
end
rule "18: Do not redact Names and Addresses if Published Information found"
when
Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "Always redact PII"
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
$cbiAuthor: RedactionEntity(type == "PII", entityType == EntityType.ENTITY)
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$cbiAuthor.addMatchedRule(1);
$cbiAuthor.setRedaction(true);
$cbiAuthor.setRedactionReason("PII found");
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type"
salience 64
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$first.removeFromGraph();
$second.removeFromGraph();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
retract($first);
retract($second);
insert(mergedEntity);
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.removeFromGraph();
retract($entity)
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 64
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 32
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$lowerRank.removeFromGraph();
retract($lowerRank);
end
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
rule "remove duplicate FileAttributes"
salience 64
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
$first: FileAttribute($label: label, $value: value)
$second: FileAttribute(this != $first, label == $label, value == $value)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
retract($second);
end
rule "28: Redact AUTHOR(S) (Vertebrate study)"
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")
)
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "33: Purity Hint"
when
Section(searchText.toLowerCase().contains("purity"))
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
end
rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
then
section.ignore("dossier_redaction");
end
rule "35: Redact signatures (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact signatures (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "43: Redact Logos (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
Set<RedactionEntity> redactionEntities = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", redactionEntities.size());
redactionEntities.forEach(redactionEntity -> insert(redactionEntity));
end

128
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json
View File

@ -6,11 +6,11 @@
"positions": [
{
"topLeft": {
"x": 173.4,
"y": 396.0
"x": 181.80199,
"y": 590.286
},
"width": -29.5,
"height": -8.1,
"width": 12.087393,
"height": 26.984985,
"page": 1
}
]
@ -20,11 +20,11 @@
"positions": [
{
"topLeft": {
"x": 189.4,
"y": 694.0
"x": 216.34381,
"y": 693.9744
},
"width": 28.0,
"height": -12.1,
"width": 26.967087,
"height": 12.087393,
"page": 1
}
]
@ -34,11 +34,11 @@
"positions": [
{
"topLeft": {
"x": 181.8,
"y": 593.3
"x": 172.43729,
"y": 395.987
},
"width": -10.1,
"height": -30.0,
"width": 26.967102,
"height": 12.087394,
"page": 1
}
]
@ -48,11 +48,11 @@
"positions": [
{
"topLeft": {
"x": 168.0,
"y": 180.1
"x": 180.08145,
"y": 210.89801
},
"width": 10.1,
"height": 30.8,
"width": 12.08743,
"height": 26.901031,
"page": 1
}
]
@ -64,11 +64,11 @@
"positions": [
{
"topLeft": {
"x": 398.8,
"y": 302.1
"x": 525.5964,
"y": 264.38696
},
"width": -27.6,
"height": -12.1,
"width": 12.087402,
"height": 26.984955,
"page": 2
}
]
@ -78,11 +78,11 @@
"positions": [
{
"topLeft": {
"x": 153.1,
"y": 258.2
"x": 178.5323,
"y": 258.195
},
"width": 25.5,
"height": -10.1,
"width": 26.967102,
"height": 12.087394,
"page": 2
}
]
@ -92,11 +92,11 @@
"positions": [
{
"topLeft": {
"x": 46.3,
"y": 304.8
"x": 56.396973,
"y": 306.31198
},
"width": 10.1,
"height": -25.5,
"width": 12.087393,
"height": 27.013977,
"page": 2
}
]
@ -106,11 +106,11 @@
"positions": [
{
"topLeft": {
"x": 523.6,
"y": 238.9
"x": 398.24277,
"y": 302.0844
},
"width": -10.1,
"height": 25.5,
"width": 26.967041,
"height": 12.087393,
"page": 2
}
]
@ -122,11 +122,11 @@
"positions": [
{
"topLeft": {
"x": 444.1,
"y": 415.4
"x": 448.598,
"y": 615.288
},
"width": 28.0,
"height": -12.1,
"width": 12.087402,
"height": 26.901001,
"page": 3
}
]
@ -136,11 +136,11 @@
"positions": [
{
"topLeft": {
"x": 448.6,
"y": 615.9
"x": 471.02884,
"y": 415.3854
},
"width": -10.1,
"height": -27.5,
"width": 26.956085,
"height": 12.087393,
"page": 3
}
]
@ -150,11 +150,11 @@
"positions": [
{
"topLeft": {
"x": 434.8,
"y": 208.2
"x": 427.23727,
"y": 117.51203
},
"width": 10.1,
"height": 27.8,
"width": 26.868073,
"height": 12.087394,
"page": 3
}
]
@ -164,11 +164,11 @@
"positions": [
{
"topLeft": {
"x": 428.2,
"y": 117.5
"x": 446.8784,
"y": 236.09802
},
"width": -29.4,
"height": -8.1,
"width": 12.087402,
"height": 26.986053,
"page": 3
}
]
@ -180,11 +180,11 @@
"positions": [
{
"topLeft": {
"x": 526.6,
"y": 487.8
"x": 431.35178,
"y": 548.7834
},
"width": 10.1,
"height": 27.9,
"width": 26.967041,
"height": 12.087393,
"page": 4
}
]
@ -194,11 +194,11 @@
"positions": [
{
"topLeft": {
"x": 212.6,
"y": 504.9
"x": 69.492966,
"y": 557.49005
},
"width": -27.5,
"height": -8.1,
"width": 12.087393,
"height": 26.901062,
"page": 4
}
]
@ -208,11 +208,11 @@
"positions": [
{
"topLeft": {
"x": 69.5,
"y": 558.5
"x": 538.6924,
"y": 515.707
},
"width": -10.1,
"height": -27.9,
"width": 12.087402,
"height": 26.899963,
"page": 4
}
]
@ -222,11 +222,11 @@
"positions": [
{
"topLeft": {
"x": 404.4,
"y": 545.8
"x": 211.64128,
"y": 504.894
},
"width": 27.1,
"height": -9.1,
"width": 26.967072,
"height": 12.087394,
"page": 4
}
]