Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
* refactored DocumentData to 4 different jsons to make reading only text possible
* wip integration of new rules into workflow
This commit is contained in:
Kilian Schuettler 2023-04-05 15:54:20 +02:00 committed by Kilian Schuettler
parent ea0181d50f
commit 59827eab81
46 changed files with 1135 additions and 1220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class AtomicPositionBlockData {
Long id;
int[] stringIdxToPositionIdx;
float[][] positions;
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java
View File

@ -15,10 +15,9 @@ public class AtomicTextBlockData {
Long id;
Long page;
String searchText;
int numberOnPage;
int start;
int end;
int[] lineBreaks;
int[] stringIdxToPositionIdx;
float[][] positions;
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java
View File

@ -1,7 +1,5 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -14,8 +12,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DocumentData {
List<PageData> pages;
List<AtomicTextBlockData> atomicTextBlocks;
PageData[] pages;
AtomicTextBlockData[] atomicTextBlocks;
AtomicPositionBlockData[] atomicPositionBlocks;
TableOfContentsData tableOfContents;
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java
View File

@ -15,5 +15,6 @@ public class PageData {
int number;
int height;
int width;
int rotation;
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java
View File

@ -15,12 +15,14 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class TableOfContentsData {
List<EntryData> entries;
@ -64,7 +66,7 @@ public class TableOfContentsData {
@Builder
public record EntryData(int[] tocId, List<EntryData> subEntries, NodeType type, Long[] atomicTextBlocks, Long[] pages, int numberOnPage, Map<String, String> properties) {
public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map<String, String> properties, List<EntryData> subEntries) {
@Override
public String toString() {
@ -80,7 +82,7 @@ public class TableOfContentsData {
sb.append(type);
sb.append(" atbs = ");
sb.append(atomicTextBlocks.length);
sb.append(atomicBlocks.length);
return sb.toString();
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java
View File

@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
import static java.lang.String.format;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@ -153,17 +151,7 @@ public class TableOfContents {
@Override
public String toString() {
return tocId + ": " + type + ".: " + buildSummary(node().buildTextBlock().getSearchText());
}
private static String buildSummary(CharSequence text) {
String[] words = text.toString().split(" ");
int bound = Math.min(words.length, 4);
List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
return String.join(" ", list);
return node().toString();
}

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java
View File

@ -25,28 +25,6 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
public class EntityNode {
public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
return EntityNode.builder()
.type(type)
.entityType(entityType)
.boundary(boundary)
.redaction(false)
.removed(false)
.ignored(false)
.resized(false)
.skipRemoveEntitiesContainedInLarger(false)
.dictionaryEntry(false)
.dossierDictionaryEntry(false)
.engines(new HashSet<>())
.references(new HashSet<>())
.matchedRule(-1)
.redactionReason("")
.legalBasis("")
.build();
}
// initial values
final Boundary boundary;
final String type;
@ -72,12 +50,34 @@ public class EntityNode {
CharSequence textAfter;
@Builder.Default
Set<PageNode> pages = new HashSet<>();
List<EntityPosition> entityPositions;
List<EntityPosition> entityPositionsPerPage;
@Builder.Default
List<SemanticNode> intersectingNodes = new LinkedList<>();
SemanticNode deepestFullyContainingNode;
public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
return EntityNode.builder()
.type(type)
.entityType(entityType)
.boundary(boundary)
.redaction(false)
.removed(false)
.ignored(false)
.resized(false)
.skipRemoveEntitiesContainedInLarger(false)
.dictionaryEntry(false)
.dossierDictionaryEntry(false)
.engines(new HashSet<>())
.references(new HashSet<>())
.matchedRule(-1)
.redactionReason("")
.legalBasis("")
.build();
}
public void addIntersectingNode(SemanticNode containingNode) {
intersectingNodes.add(containingNode);
@ -93,13 +93,13 @@ public class EntityNode {
}
public List<EntityPosition> getEntityPositions() {
public List<EntityPosition> getEntityPositionsPerPage() {
if (entityPositions == null || entityPositions.isEmpty()) {
entityPositions = deepestFullyContainingNode.buildTextBlock().getEntityPositions(boundary);
if (entityPositionsPerPage == null || entityPositionsPerPage.isEmpty()) {
entityPositionsPerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary);
}
return entityPositions;
return entityPositionsPerPage;
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java
View File

@ -28,10 +28,10 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
@ -40,7 +40,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.ImageSortService;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@ -48,20 +47,14 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class DocumentGraphFactory {
public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
private final ImageSortService imageSortService;
public DocumentGraph buildDocumentGraph(Document document) {
ImageSortService.SortedImages sortedImages = imageSortService.sortImagesIntoStructure(document);
TextBlockFactory textBlockFactory = new TextBlockFactory();
Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
@ -121,13 +114,13 @@ public class DocumentGraphFactory {
}
private List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
return pageBlocks.stream()
.filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
.filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
.filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock)
.filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
.filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
.map(abstractTextContainer -> (TextBlock) abstractTextContainer)
.toList();
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java
View File

@ -7,13 +7,10 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@Service
public class SearchTextWithTextPositionFactory {
public static final int HEIGHT_PADDING = 2;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java
View File

@ -47,7 +47,7 @@ public class FooterNode implements SemanticNode {
@Override
public String toString() {
return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.getSearchText();
return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.buildSummary();
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java
View File

@ -47,7 +47,7 @@ public class HeaderNode implements SemanticNode {
@Override
public String toString() {
return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.getSearchText();
return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.buildSummary();
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java
View File

@ -47,7 +47,7 @@ public class HeadlineNode implements SemanticNode {
@Override
public String toString() {
return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.toString();
return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.buildSummary();
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
@ -9,9 +9,7 @@ import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java
View File

@ -1,52 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
public enum NodeType {
SECTION {
public String toString() {
return "Section";
}
},
HEADLINE {
public String toString() {
return "Headline";
}
},
PARAGRAPH {
public String toString() {
return "Paragraph";
}
},
TABLE {
public String toString() {
return "Table";
}
},
TABLE_CELL {
public String toString() {
return "Cell";
}
},
IMAGE {
public String toString() {
return "Image";
}
},
HEADER {
public String toString() {
return "Header";
}
},
FOOTER {
public String toString() {
return "Footer";
}
}
SECTION,
HEADLINE,
PARAGRAPH,
TABLE,
TABLE_CELL,
IMAGE,
HEADER,
FOOTER
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java
View File

@ -5,7 +5,6 @@ import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java
View File

@ -45,7 +45,7 @@ public class ParagraphNode implements SemanticNode {
@Override
public String toString() {
return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString();
return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.buildSummary();
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java
View File

@ -49,7 +49,7 @@ public class SectionNode implements SemanticNode {
@Override
public String toString() {
return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().getSearchText();
return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary();
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java
View File

@ -168,6 +168,16 @@ public interface SemanticNode {
}
/**
* @param strings A List of Strings which the TextBlock might contain
* @return true, if this node's TextBlock contains any of the strings
*/
default boolean containsAnyString(List<String> strings) {
return strings.stream().anyMatch(this::containsString);
}
/**
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
* It sets the fields accordingly and recursively calls this function on all its children.
@ -221,7 +231,8 @@ public interface SemanticNode {
/**
* If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children
* If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
* If called on the Document, it will return the cropbox of each page
*
* @return Rectangle2D fully encapsulating this Node for each page.
*/
@ -237,6 +248,8 @@ public interface SemanticNode {
/**
* TODO this does not yet work for sections spanning multiple columns
*
* @param bBoxPerPage initial empty BoundingBox
* @return The union of the BoundingBoxes of all children
*/

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java
View File

@ -73,7 +73,7 @@ public class TableCellNode implements SemanticNode {
@Override
public String toString() {
return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().getSearchText();
return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary();
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java
View File

@ -67,7 +67,7 @@ public class TableNode implements SemanticNode {
@Override
public String toString() {
return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().getSearchText();
return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java
View File

@ -112,7 +112,7 @@ public class AtomicTextBlock implements TextBlock {
}
public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
.stream()

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java
View File

@ -139,23 +139,23 @@ public class ConcatenatedTextBlock implements TextBlock {
@Override
public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
if (textBlocks.size() == 1) {
return textBlocks.get(0).getEntityPositions(stringBoundary);
return textBlocks.get(0).getEntityPositionsPerPage(stringBoundary);
}
AtomicTextBlock firstTextBlock = textBlocks.get(0);
List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
positions.addAll(textBlock.getEntityPositions(textBlock.getBoundary()));
positions.addAll(textBlock.getEntityPositionsPerPage(textBlock.getBoundary()));
}
AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
positions.addAll(lastTextBlock.getEntityPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
return mergeEntityPositionsWithSamePageNode(positions);
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
import static java.lang.String.format;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -27,6 +29,7 @@ public interface TextBlock extends CharSequence {
int getPreviousLinebreak(int fromIndex);
List<Integer> getLineBreaks();
@ -36,16 +39,18 @@ public interface TextBlock extends CharSequence {
List<Rectangle2D> getPositions(Boundary stringBoundary);
List<EntityPosition> getEntityPositions(Boundary stringBoundary);
List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary);
int numberOfLines();
default int indexOf(String searchTerm) {
return indexOf(searchTerm, getBoundary().start());
}
default Set<PageNode> getPages() {
return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet());
@ -89,6 +94,16 @@ public interface TextBlock extends CharSequence {
}
default String buildSummary() {
String[] words = getSearchText().split(" ");
int bound = Math.min(words.length, 4);
List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
return String.join(" ", list);
}
@Override
default CharSequence subSequence(int start, int end) {

33
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java
View File

@ -7,13 +7,14 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
@ -30,9 +31,21 @@ public class DocumentDataMapper {
.distinct()
.map(this::toAtomicTextBlockData)
.toList();
List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicPositionBlockData)
.toList();
List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build();
return DocumentData.builder()
.atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
.atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
.pages(pageData.toArray(new PageData[0]))
.tableOfContents(tableOfContentsData)
.build();
}
@ -65,7 +78,6 @@ public class DocumentDataMapper {
.type(entry.type())
.atomicTextBlocks(atomicTextBlocks)
.pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
.numberOnPage(entry.node().getNumberOnPage())
.properties(properties)
.build();
}
@ -77,9 +89,9 @@ public class DocumentDataMapper {
}
private PageData toPageData(PageNode pageNode) {
private PageData toPageData(PageNode p) {
return PageData.builder().height(pageNode.getHeight()).width(pageNode.getWidth()).number(pageNode.getNumber()).build();
return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
}
@ -89,11 +101,20 @@ public class DocumentDataMapper {
.id(atomicTextBlock.getId())
.page(atomicTextBlock.getPage().getNumber().longValue())
.searchText(atomicTextBlock.getSearchText())
.numberOnPage(atomicTextBlock.getNumberOnPage())
.start(atomicTextBlock.getBoundary().start())
.end(atomicTextBlock.getBoundary().end())
.lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
.stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
.build();
}
private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
return AtomicPositionBlockData.builder()
.id(atomicTextBlock.getId())
.positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
.stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
.build();
}

50
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java
View File

@ -17,6 +17,7 @@ import org.apache.commons.lang3.NotImplementedException;
import org.springframework.stereotype.Service;
import com.google.common.primitives.Ints;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
@ -24,10 +25,10 @@ import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsDa
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
@ -45,14 +46,19 @@ public class DocumentGraphMapper {
public DocumentGraph toDocumentGraph(DocumentData documentData) {
Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks());
Context context = new Context(documentData,
new TableOfContents(),
new LinkedList<>(),
new LinkedList<>(),
Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
context.pages.addAll(documentData.getPages().stream().map(this::buildPage).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList());
context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context));
DocumentGraph documentGraph = DocumentGraph.builder()
.numberOfPages(documentData.getPages().size())
.numberOfPages(documentData.getPages().length)
.pages(new HashSet<>(context.pages))
.tableOfContents(context.tableOfContents)
.build();
@ -82,7 +88,7 @@ public class DocumentGraphMapper {
};
if (node.isTerminal()) {
TextBlock textBlock = toTextBlock(entryData.atomicTextBlocks(), context, node);
TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node);
node.setTerminalTextBlock(textBlock);
}
List<Integer> tocId = Arrays.stream(entryData.tocId()).boxed().toList();
@ -109,7 +115,7 @@ public class DocumentGraphMapper {
private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
return entryData.atomicTextBlocks().length > 0;
return entryData.atomicBlocks().length > 0;
}
@ -165,34 +171,42 @@ public class DocumentGraphMapper {
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds)
.map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), parent, context))
.map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)),
context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)),
parent,
context))
.collect(new TextBlockCollector());
}
private PageNode buildPage(PageData p) {
return PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
return PageNode.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
}
private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, SemanticNode parent, Context context) {
private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, AtomicPositionBlockData atomicPositionBlockData, SemanticNode parent, Context context) {
return AtomicTextBlock.builder()
.id(atomicTextBlockData.getId())
.numberOnPage(atomicTextBlockData.getNumberOnPage())
.page(getPage(atomicTextBlockData.getPage(), context))
.searchText(atomicTextBlockData.getSearchText())
.boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
.searchText(atomicTextBlockData.getSearchText())
.lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
.positions(Arrays.stream(atomicTextBlockData.getPositions())
.map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
.toList())
.stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx()))
.stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx()))
.positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
.parent(parent)
.build();
}
private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
}
private PageNode getPage(Long pageIndex, Context context) {
return context.pages.stream()
@ -202,7 +216,13 @@ public class DocumentGraphMapper {
}
record Context(DocumentData documentData, TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, List<AtomicTextBlockData> atomicTextBlockData) {
record Context(
DocumentData documentData,
TableOfContents tableOfContents,
List<PageNode> pages,
List<SectionNode> sections,
List<AtomicTextBlockData> atomicTextBlockData,
List<AtomicPositionBlockData> atomicPositionBlockData) {
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java
View File

@ -5,8 +5,8 @@ import static com.iqser.red.service.redaction.v1.server.document.graph.factory.R
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.LinkedList;
import java.util.List;
@ -43,7 +43,7 @@ public class CharSequenceSearchUtils {
while (matcher.find()) {
boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
}
return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
return boundaries.stream().filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
}

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -1,7 +1,8 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
import java.util.Collections;
import java.util.LinkedList;
@ -30,7 +31,7 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class EntityCreationService {
private final EntityTextEnrichmentService entityEnrichmentService;
private final EntityEnrichmentService entityEnrichmentService;
public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
@ -52,7 +53,7 @@ public class EntityCreationService {
entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
}
return entityBoundaries.stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
@ -62,19 +63,32 @@ public class EntityCreationService {
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
.filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary))
.map(bounds -> byBoundary(bounds, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
public Set<EntityNode> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, false);
return searchImplementation.getBoundaries(textBlock, node.getBoundary())
.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
List<Boundary> boundaries = findBoundariesByString(string, textBlock);
return boundaries.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
@ -103,7 +117,7 @@ public class EntityCreationService {
}
public EntityNode createMergedEntity(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
public EntityNode byEntities(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
throw new IllegalArgumentException("Provided entities can not be merged!");
@ -121,6 +135,12 @@ public class EntityCreationService {
}
public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
}
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
try {

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java
View File

@ -1,11 +1,13 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@ -14,17 +16,16 @@ import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class EntityTextEnrichmentService {
public class EntityEnrichmentService {
private final RedactionServiceSettings redactionServiceSettings;
public EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) {
public void enrichEntity(EntityNode entity, TextBlock textBlock) {
entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
return entity;
}
@ -64,7 +65,7 @@ public class EntityTextEnrichmentService {
}
private String concatWordsBefore(List<String> words, boolean endWithSpace) {
private static String concatWordsBefore(List<String> words, boolean endWithSpace) {
StringBuilder sb = new StringBuilder();
@ -77,7 +78,7 @@ public class EntityTextEnrichmentService {
}
private String concatWordsAfter(List<String> words, boolean startWithSpace) {
private static String concatWordsAfter(List<String> words, boolean startWithSpace) {
StringBuilder sb = new StringBuilder();
@ -89,4 +90,27 @@ public class EntityTextEnrichmentService {
return startWithSpace ? " " + result : result;
}
public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
entityNode.setMatchedRule(matchedRule);
}
if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
entityNode.setRedactionReason(redactionReason);
}
if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
entityNode.setLegalBasis(legalBasis);
}
if (engine != null) {
entityNode.addEngine(engine);
}
}
public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
}
}

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java
View File

@ -1,32 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import java.util.Collection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
public class EntityFieldSetter {
public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
entityNode.setMatchedRule(matchedRule);
}
if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
entityNode.setRedactionReason(redactionReason);
}
if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
entityNode.setLegalBasis(legalBasis);
}
if (engine != null) {
entityNode.addEngine(engine);
}
}
public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java
View File

@ -92,7 +92,7 @@ public class Dictionary {
if (localAccessMap.get(type).getLocalEntries() == null) {
throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
}
if (StringUtils.isEmpty(value) && value.length() < 3) {
if (StringUtils.isEmpty(value)) {
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
}
localAccessMap.get(type).getLocalEntries().add(value);

59
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -26,11 +26,15 @@ import org.kie.api.runtime.rule.QueryResults;
import org.kie.api.runtime.rule.QueryResultsRow;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import io.micrometer.core.annotation.Timed;
@ -77,6 +81,8 @@ public class DroolsExecutionService {
});
}
private final EntityCreationService entityCreationService;
public KieContainer getKieContainer(String dossierTemplateId) {
@ -90,18 +96,46 @@ public class DroolsExecutionService {
@Timed("redactmanager_executeRules")
public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
public List<FileAttribute> executeRules(KieContainer kieContainer, DocumentGraph document, Dictionary dictionary, List<FileAttribute> fileAttributes) {
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("dictionary", dictionary);
sections.forEach(kieSession::insert);
paragraphs.forEach(kieSession::insert);
document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
document.getPages().forEach(kieSession::insert);
fileAttributes.forEach(kieSession::insert);
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
kieSession.fireAllRules();
List<Entity> entities = getEntities(kieSession);
kieSession.dispose();
return entities;
return getFileAttributes(kieSession);
}
@Timed("redactmanager_executeRules")
public List<FileAttribute> executeRules(KieContainer kieContainer,
DocumentGraph document,
List<SectionNode> sectionsToReanalyze,
Dictionary dictionary,
List<FileAttribute> fileAttributes) {
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("dictionary", dictionary);
document.getEntities().forEach(kieSession::insert);
sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
document.getPages().forEach(kieSession::insert);
fileAttributes.forEach(kieSession::insert);
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
kieSession.fireAllRules();
return getFileAttributes(kieSession);
}
@ -119,6 +153,17 @@ public class DroolsExecutionService {
}
public List<FileAttribute> getFileAttributes(KieSession kieSession) {
List<FileAttribute> fileAttributes = new LinkedList<>();
QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
for (QueryResultsRow resultsRow : entitiesResult) {
fileAttributes.add((FileAttribute) resultsRow.get("$result"));
}
return fileAttributes;
}
public List<Entity> getEntities(KieSession ks) {
List<Entity> entities = new LinkedList<>();

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -52,7 +52,7 @@ public class RedactionLogCreatorService {
// Duplicates can exist due table extraction columns over multiple rows.
for (EntityPosition entityPosition : entityNode.getEntityPositions()) {
for (EntityPosition entityPosition : entityNode.getEntityPositionsPerPage()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId);
@ -61,7 +61,6 @@ public class RedactionLogCreatorService {
}
processedIds.add(entityPosition.getId());
redactionLogEntry.setId(entityPosition.getId());
List<Rectangle> rectanglesPerLine = entityPosition.getRectanglePerLine()
@ -70,6 +69,7 @@ public class RedactionLogCreatorService {
.toList();
redactionLogEntry.setPositions(rectanglesPerLine);
redactionLogEntities.add(redactionLogEntry);
}
return redactionLogEntities;
@ -79,7 +79,7 @@ public class RedactionLogCreatorService {
private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId())));
entity.getReferences().forEach(ref -> ref.getEntityPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
@ -91,7 +91,7 @@ public class RedactionLogCreatorService {
.isHint(isHint(entity.getType(), dossierTemplateId))
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(entity.getDeepestFullyContainingNode().getTocId().toString())
.section(entity.getDeepestFullyContainingNode().toString())
.sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
.matchedRule(entity.getMatchedRule())
.isDictionaryEntry(entity.isDictionaryEntry())

69
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -33,11 +33,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
@ -81,6 +85,9 @@ public class AnalyzeService {
ImageService imageService;
ImportedRedactionService importedRedactionService;
SectionFinder sectionFinder;
DocumentGraphFactory documentGraphFactory;
DocumentDataMapper documentDataMapper;
DocumentGraphMapper documentGraphMapper;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@ -109,6 +116,8 @@ public class AnalyzeService {
throw new RedactionException(e);
}
DocumentGraph documentGraph = documentGraphFactory.buildDocumentGraph(classifiedDoc);
List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
@ -123,8 +132,8 @@ public class AnalyzeService {
sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
sectionText.getSectionAreas())));
log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
@ -144,11 +153,11 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
// not yet ready for reanalysis
if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
}
@ -156,28 +165,31 @@ public class AnalyzeService {
new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
Set<Integer> sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest);
if (sectionsToReanalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
if (sectionsToReanalyseIds.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
}
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
List<SectionText> reanalysisSections = text.getSectionTexts()
List<SectionNode> sectionsToReAnalyse = documentGraph.getMainSections()
.stream()
.filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
.filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0)))
.collect(Collectors.toList());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
: new NerEntities();
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
@ -185,10 +197,10 @@ public class AnalyzeService {
newRedactionLogEntries,
false);
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, pageEntities.getAddedFileAttributes());
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
}
@ -196,7 +208,7 @@ public class AnalyzeService {
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
@ -209,12 +221,13 @@ public class AnalyzeService {
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
redactionLogEntries,
convert(legalBasis),
@ -223,21 +236,21 @@ public class AnalyzeService {
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog.getRedactionLogEntry(),
true);
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false, pageEntities.getAddedFileAttributes());
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionary.getVersion(), false, addedFileAttributes);
}
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
RedactionLog redactionLog,
Text text,
int numberOfPages,
DictionaryVersion dictionaryVersion,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes) {
@ -255,13 +268,13 @@ public class AnalyzeService {
long duration = System.currentTimeMillis() - startTime;
redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration);
redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(text.getNumberOfPages())
.numberOfPages(numberOfPages)
.hasUpdates(redactionLogChange.isHasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -19,13 +19,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import io.micrometer.core.annotation.Timed;
@ -39,8 +38,9 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
class SectionFinder {
@Timed("redactmanager_findSectionsToReanalyse")
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, DocumentGraph documentGraph, AnalyzeRequest analyzeRequest) {
long start = System.currentTimeMillis();
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@ -59,10 +59,10 @@ class SectionFinder {
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
true);
for (SectionText sectionText : text.getSectionTexts()) {
for (SectionNode section : documentGraph.getMainSections()) {
if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
sectionsToReanalyse.add(sectionText.getSectionNumber());
if (dictionaryIncrementsSearch.atLeastOneMatches(section.buildTextBlock().getSearchText())) {
sectionsToReanalyse.add(section.getTocId().get(0));
}
}
@ -72,6 +72,7 @@ class SectionFinder {
return sectionsToReanalyse;
}
private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
if (manualRedactions == null) {
@ -85,6 +86,7 @@ class SectionFinder {
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
}
private static Image convert(RedactionLogEntry entry) {
Rectangle position = entry.getPositions().get(0);
@ -98,4 +100,5 @@ class SectionFinder {
.hasTransparency(entry.isImageHasTransparency())
.build();
}
}

313
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -1,39 +1,22 @@
package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -47,283 +30,47 @@ import lombok.extern.slf4j.Slf4j;
public class EntityRedactionService {
DroolsExecutionService droolsExecutionService;
EntityFinder entityFinder;
RedactionServiceSettings redactionServiceSettings;
EntityCreationService entityCreationService;
public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
NerEntities nerEntities) {
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
long start = System.currentTimeMillis();
FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size());
if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
//AnalyzeRequest provides immutable list.
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes());
analyzeRequest.setFileAttributes(mergedFileAttributes);
}
long start1 = System.currentTimeMillis();
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
dictionary,
kieContainer,
analyzeRequest,
true,
hintsPerSectionNumber,
imagesPerPage,
nerEntities);
EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1);
}
Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
//EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start);
return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, dictionary, analyzeRequest.getFileAttributes());
return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
}
private FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
Dictionary dictionary,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
Map<Integer, Set<Image>> imagesPerPage,
NerEntities nerEntities) {
public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
DocumentGraph documentGraph,
List<SectionNode> sectionsToReanalyze,
KieContainer kieContainer,
AnalyzeRequest analyzeRequest,
NerEntities nerEntities) {
long start = System.currentTimeMillis();
List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
documentGraph,
sectionsToReanalyze,
dictionary,
analyzeRequest,
local,
hintsPerSectionNumber,
nerEntities);
System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start);
Set<FileAttribute> addedFileAttributes = new HashSet<>();
Set<Entity> entities = new HashSet<>();
long start1 = System.currentTimeMillis();
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
if (!addedFileAttributes.isEmpty()) {
//Section.Builder provides immutable list.
List<FileAttribute> mergedFileAttributes = new ArrayList<>();
mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
mergedFileAttributes.addAll(addedFileAttributes);
sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
}
if (sectionSearchableTextPair.getSection() == null) {
return;
}
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
var entriesWithoutSurroundingText = analysedSection.getEntities()
.stream()
.filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
.collect(Collectors.toSet());
addSurroundingText(dictionary, sectionSearchableTextPair.getSearchableText(), sectionSearchableTextPair.getCellStarts(), entriesWithoutSurroundingText);
entities.addAll(analysedSection.getEntities());
if (!local) {
for (Image image : analysedSection.getImages()) {
imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
}
addLocalValuesToDictionary(analysedSection, dictionary);
}
});
System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1);
return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
analyzeRequest.getFileAttributes());
return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
}
private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List<Integer> cellStarts, Set<Entity> entriesWithoutSurroundingText) {
public void addDictionaryEntities(Dictionary dictionary, DocumentGraph documentGraph) {
if (cellStarts != null && !cellStarts.isEmpty()) {
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
searchableText,
dictionary,
cellStarts,
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
} else {
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
searchableText,
dictionary,
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
}
dictionary.getDictionaryModels().forEach(dictionaryModel -> addDictionaryModelEntities(dictionaryModel, documentGraph));
}
private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
Dictionary dictionary,
AnalyzeRequest analyzeRequest,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
NerEntities nerEntities) {
private void addDictionaryModelEntities(DictionaryModel dictionaryModel, DocumentGraph documentGraph) {
return reanalysisSections.stream().map(reanalysisSection -> {
long start = System.currentTimeMillis();
Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
reanalysisSection.getHeadline(),
reanalysisSection.getSectionNumber(),
dictionary,
local,
nerEntities,
reanalysisSection.getCellStarts(),
analyzeRequest.getManualRedactions());
addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities());
if (!local && analyzeRequest.getManualRedactions() != null) {
var approvedForceRedactions = analyzeRequest.getManualRedactions()
.getForceRedactions()
.stream()
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
.filter(fr -> fr.getRequestDate() != null)
.collect(Collectors.toList());
// only approved id removals, that haven't been forced back afterwards
var idsToRemove = analyzeRequest.getManualRedactions()
.getIdsToRemove()
.stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream()
.noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
.isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId)
.collect(Collectors.toSet());
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
if (idsToRemove.contains(imageId)) {
image.setIgnored(true);
}
}
}
entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
if (idsToRemove.contains(ps.getId())) {
entity.setIgnored(true);
}
}));
}
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
}).collect(Collectors.toList());
}
private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
AnalyzeRequest analyzeRequest,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
SectionText reanalysisSection,
Entities entities) {
return new SectionSearchableTextPair(Section.builder()
.isLocal(false)
.dictionaryTypes(dictionary.getTypes())
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
.nerEntities(entities.getNerEntities())
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
.searchText(reanalysisSection.getSearchableText().toString())
.headline(reanalysisSection.getHeadline())
.sectionNumber(reanalysisSection.getSectionNumber())
.tabularData(reanalysisSection.getTabularData())
.searchableText(reanalysisSection.getSearchableText())
.dictionary(dictionary)
.images(reanalysisSection.getImages())
.sectionAreas(reanalysisSection.getSectionAreas())
.fileAttributes(analyzeRequest.getFileAttributes())
.manualRedactions(analyzeRequest.getManualRedactions())
.isInTable(reanalysisSection.isTable())
.build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
}
private Map<Integer, List<Entity>> convertToEntitiesPerPage(Set<Entity> entities) {
Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
for (Entity entity : entities) {
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()).add(entityPositionSequence);
}
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
.add(new Entity(entity.getWord(),
entity.getType(),
entity.isRedaction(),
entity.getRedactionReason(),
entry.getValue(),
entity.getHeadline(),
entity.getMatchedRule(),
entity.getSectionNumber(),
entity.getLegalBasis(),
entity.isDictionaryEntry(),
entity.getTextBefore(),
entity.getTextAfter(),
entity.getStart(),
entity.getEnd(),
entity.isDossierDictionaryEntry(),
entity.getEngines(),
entity.getReferences(),
entity.getEntityType()));
}
}
return entitiesPerPage;
}
private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
entities.forEach(entity -> {
if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
}
});
return hintsPerSectionNumber;
}
private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {
analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
if (dictionary.getLocalAccessMap().get(key) == null) {
log.warn("Dictionary {} is null", key);
}
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
log.warn("Dictionary {} localEntries is null", key);
}
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
}));
entityCreationService.bySearchImplementation(dictionaryModel.getEntriesSearch(), dictionaryModel.getType(), EntityType.ENTITY, documentGraph);
entityCreationService.bySearchImplementation(dictionaryModel.getFalsePositiveSearch(), dictionaryModel.getType(), EntityType.FALSE_POSITIVE, documentGraph);
entityCreationService.bySearchImplementation(dictionaryModel.getFalseRecommendationsSearch(), dictionaryModel.getType(), EntityType.FALSE_RECOMMENDATION, documentGraph);
}
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java
View File

@ -4,6 +4,7 @@ import java.util.Set;
import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@ -31,22 +32,38 @@ public final class SeparatorUtils {
}
public static boolean isWhiteSpacesOrSeparatorsOnly(TextBlock textBlock, Boundary boundary) {
public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) {
return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary);
String stringWithoutWhiteSpace = textBlock.subSequence(boundary).toString().replace(" ", "");
int numberOfSeparators = 0;
for (int i = 0; i < stringWithoutWhiteSpace.length(); i++) {
if (isSeparator(stringWithoutWhiteSpace.charAt(i))) {
numberOfSeparators++;
}
}
return stringWithoutWhiteSpace.length() == numberOfSeparators;
}
private static boolean validateEnd(CharSequence charSequence, Boundary boundary) {
public static boolean boundaryIsSurroundedBySeparators(TextBlock textBlock, Boundary boundary) {
return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1));
return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary);
}
private static boolean validateStart(CharSequence charSequence, Boundary boundary) {
private static boolean validateEnd(TextBlock textBlock, Boundary boundary) {
return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start()));
return boundary.end() == textBlock.getBoundary().end() ||//
SeparatorUtils.isSeparator(textBlock.charAt(boundary.end())) ||//
SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1));
}
private static boolean validateStart(TextBlock textBlock, Boundary boundary) {
return boundary.start() == textBlock.getBoundary().start() ||//
SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) ||//
SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()));
}
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
@ -55,7 +56,9 @@ public class RedactionStorageService {
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
return storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactions.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available.");
return null;
@ -88,6 +91,18 @@ public class RedactionStorageService {
}
@Timed("redactmanager_getDocumentGraph")
public DocumentData getDocumentData(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), DocumentData.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("Text not available.");
return null;
}
}
@Timed("redactmanager_getNerEntities")
public NerEntities getNerEntities(String dossierId, String fileId) {

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList;
import java.awt.Color;
@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
@SneakyThrows
public void testDroolsOnDocumentGraph() {
String filename = "files/new/crafted document";
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@ -217,7 +217,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.stream()
.filter(entityNode -> !entityNode.isRemoved())
.filter(EntityNode::isRedaction)
.flatMap(entityNode -> entityNode.getEntityPositions().stream())
.flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();
@ -231,7 +231,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.stream()
.filter(entityNode -> !entityNode.isRemoved())
.filter(entityNode -> !entityNode.isRedaction())
.flatMap(entityNode -> entityNode.getEntityPositions().stream())
.flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();
@ -255,7 +255,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
TextBlock textBlock = documentGraph.getTextBlock();
searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
.stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
.map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
.forEach(foundEntities::add);
}

19
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java
View File

@ -5,7 +5,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
@ -33,7 +37,7 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
@SneakyThrows
public void testGraphMapping() {
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
String filename = "files/new/crafted document";
prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@ -41,7 +45,18 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
DocumentData documentData = documentDataMapper.toDocumentData(document);
storageService.storeJSONObject(TenantContext.getTenantId(), filename + ".json", documentData);
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", documentData.getTableOfContents());
PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", PageData[].class);
AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", AtomicTextBlockData[].class);
AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
filename + "_POSITIONS" + ".json",
AtomicPositionBlockData[].class);
TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class);
DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
assert document.toString().equals(newDocumentGraph.toString());

119
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl
View File

@ -2,7 +2,7 @@ package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
import java.util.List;
@ -131,6 +131,18 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
entities.forEach(entity -> insert(entity));
end
rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
when
$section : SectionNode(containsString("BL") || containsString("CT"))
then
Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
// --------------------------------------- PII rules -------------------------------------------------------------------
@ -182,43 +194,106 @@ rule "13: Redact Emails by RegEx (Vertebrate study)"
entities.forEach(entity -> insert(entity));
end
rule "14: Redact contact information (Non vertebrate study)"
rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode()
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE);
entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "15: Redact contact information (Vertebrate study)"
rule "15: Redact line after contact information (Vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode()
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE);
Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "16: Redact Phone and Fax by RegEx"
rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "17: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "18: Redact Phone and Fax by RegEx"
when
$section: SectionNode(containsString("Contact") ||
@ -232,6 +307,6 @@ rule "16: Redact Phone and Fax by RegEx"
containsString("Fer"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE);
setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end

5
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl
View File

@ -22,6 +22,9 @@ global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
rule "merge intersecting Entities of same type"
salience 100
@ -32,7 +35,7 @@ rule "merge intersecting Entities of same type"
then
$first.removeFromGraph();
$second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity);
retract($first);
retract($second);

601
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1,367 +1,390 @@
package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
global Section section
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
// --------------------------------------- AI rules -------------------------------------------------------------------
import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
rule "0: Add CBI_author from ai"
when
Section(aiMatchesType("CBI_author"))
then
section.addAiEntities("CBI_author", "CBI_author");
import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
rule "merge intersecting Entities of same type"
salience 100
when
Section(aiMatchesType("ORG"))
$first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
$first.removeFromGraph();
$second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity);
retract($first);
retract($second);
end
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 100
when
Section(aiMatchesType("STREET"))
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
entity.removeFromGraph();
retract(entity);
end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 100
when
Section(aiMatchesType("CITY"))
$falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 100
when
$entity: EntityNode($type: type, entityType == EntityType.ENTITY)
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 100
when
$higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
then
$lowerRank.removeFromGraph();
retract($lowerRank);
end
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
entityNodes.forEach(entityNode -> insert(entityNode));
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "2: Redact CBI Authors (Vertebrate study)"
no-loop true
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "3: Don't redact CBI Address (Non vertebrate study)"
no-loop true
rule "3: Redact not CBI Address (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "4: Redact CBI Address (Vertebrate study)"
no-loop true
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
insert(entity);
end
rule "5: Do not redact genitive CBI_author"
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
Section(matchesType("CBI_author"))
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
end
rule "7: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
end
rule "8: Add recommendation for Addresses in Test Organism sections"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
rule "9: Add recommendation for Addresses in Test Animals sections"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species:") && containsString("Source:"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
$section : SectionNode(containsString("BL") || containsString("CT"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
then
section.recommendLineAfter("Source", "CBI_address");
end
rule "18: Do not redact Names and Addresses if Published Information found"
when
Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
no-loop true
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "23: Redact contact information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "24: Redact contact information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "28: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")
)
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "33: Redact study director abbreviation"
when
Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
then
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "34: Purity Hint"
when
Section(searchText.toLowerCase().contains("purity"))
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
$entity.setRedaction(true);
setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "35: Redact signatures (Non vertebrate study)"
rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
no-loop true
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "36: Redact signatures (Vertebrate study)"
rule "12: Redact Emails by RegEx (Non vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "13: Redact Emails by RegEx (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("@"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 13, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "43: Redact Logos (Vertebrate study)"
rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "15: Redact line after contact information (Vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "17: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "18: Redact Phone and Fax by RegEx"
when
$section: SectionNode(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
containsString("Mobile") ||
containsString("Fel") ||
containsString("Fer"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl
View File

@ -1 +0,0 @@
rules.drl

619
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl
View File

@ -1,431 +1,390 @@
package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
global Section section
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
// --------------------------------------- AI rules -------------------------------------------------------------------
import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
rule "0: Add CBI_author from ai"
when
Section(aiMatchesType("CBI_author"))
then
section.addAiEntities("CBI_author", "CBI_author",dictionary);
import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
rule "merge intersecting Entities of same type"
salience 100
when
Section(aiMatchesType("ORG"))
$first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
$first.removeFromGraph();
$second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity);
retract($first);
retract($second);
end
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 100
when
Section(aiMatchesType("STREET"))
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
$entity.removeFromGraph();
retract($entity);
end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 100
when
Section(aiMatchesType("CITY"))
$falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 100
when
$entity: EntityNode($type: type, entityType == EntityType.ENTITY)
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 100
when
$higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
then
$lowerRank.removeFromGraph();
retract($lowerRank);
end
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
entityNodes.forEach(entityNode -> insert(entityNode));
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "2: Redact CBI Authors (Vertebrate study)"
no-loop true
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "3: Don't redact CBI Address (Non vertebrate study)"
no-loop true
rule "3: Redact not CBI Address (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "4: Redact CBI Address (Vertebrate study)"
no-loop true
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
insert(entity);
end
rule "5: Do not redact genitive CBI_author"
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
Section(matchesType("CBI_author"))
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
end
rule "7: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
end
rule "8: Add recommendation for Addresses in Test Organism sections"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
rule "9: Add recommendation for Addresses in Test Animals sections"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species:") && containsString("Source:"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
/* Syngenta specific laboratory rule */
rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
when
Section(searchText.contains("CT") || searchText.contains("BL"))
$section : SectionNode(containsString("BL") || containsString("CT"))
then
section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address");
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
then
section.recommendLineAfter("Source", "CBI_address");
end
rule "18: Do not redact Names and Addresses if Published Information found"
when
Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
no-loop true
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.setRedaction(true);
setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
no-loop true
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
$entity.setRedaction(true);
setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
rule "12: Redact Emails by RegEx (Non vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "13: Redact Emails by RegEx (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("@"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 13, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "23: Redact contact information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Phone:")
|| text.contains("Fax:")
|| text.contains("Tel.:")
|| text.contains("Tel:")
|| text.contains("E-mail:")
|| text.contains("Email:")
|| text.contains("e-mail:")
|| text.contains("E-mail address:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| text.contains("Telephone number:")
|| text.contains("Telephone No:")
|| text.contains("Fax number:")
|| text.contains("Telephone:")
|| text.contains("Phone No.")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
rule "24: Redact contact information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Phone:")
|| text.contains("Fax:")
|| text.contains("Tel.:")
|| text.contains("Tel:")
|| text.contains("E-mail:")
|| text.contains("Email:")
|| text.contains("e-mail:")
|| text.contains("E-mail address:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| text.contains("Telephone number:")
|| text.contains("Telephone No:")
|| text.contains("Fax number:")
|| text.contains("Telephone:")
|| text.contains("Phone No.")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Ph.")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Cell")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Ph.")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Cell")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "15: Redact line after contact information (Vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
rule "28: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: SectionNode(containsString($string))
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
rule "17: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "18: Redact Phone and Fax by RegEx"
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")
)
$section: SectionNode(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
containsString("Mobile") ||
containsString("Fel") ||
containsString("Fer"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "33: Purity Hint"
when
Section(searchText.toLowerCase().contains("purity"))
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
end
rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
then
section.ignore("dossier_redaction");
end
rule "35: Redact signatures (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact signatures (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "43: Redact Logos (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end

35
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -1,35 +0,0 @@
package drools
import static java.lang.String.format;
import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import java.util.Set;
global DocumentGraph document
rule "dummy"
when
then
System.out.println("This actually works");
end
rule "1: Redact CBI_author"
when
//FileAttribute(label == "Vertebrate Study" && (value == "Yes"))
$entity: EntityNode(type == "CBI_author")
then
System.out.println("Entity found");
modify($entity){
$entity.setRedact(true)
}
end