Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-8049' into 'master'

Browse Source 
RED-8049 - Change TextEntity id to be calculated based on position

Closes RED-8049

See merge request redactmanager/redaction-service!255
This commit is contained in:
Andrei Isvoran 2024-02-01 11:30:47 +01:00
commit e43e9ace68
54 changed files with 988 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.redaction.v1.server.model;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import lombok.Builder;
import lombok.Getter;
@ -12,6 +12,6 @@ import lombok.Setter;
public class ClosestEntity {
private double distance;
private TextRange textRange;
private TextEntity textEntity;
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java
View File

@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.model;
import java.util.List;
import java.util.PriorityQueue;
import java.util.UUID;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@ -69,6 +71,7 @@ public class ManualEntity implements IEntity {
.build();
}
public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
@ -90,6 +93,19 @@ public class ManualEntity implements IEntity {
}
public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
return ManualEntity.builder()
.id(UUID.randomUUID().toString())
.value(manualResizeRedaction.getValue())
.entityPosition(rectangleWithPages)
.entityType(EntityType.ENTITY)
.manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
.build();
}
@Override
public TextRange getTextRange() {
@ -103,6 +119,7 @@ public class ManualEntity implements IEntity {
return getManualOverwrite().getType().orElse(type);
}
private static EntityType getEntityType(EntryType entryType) {
switch (entryType) {
@ -124,7 +141,9 @@ public class ManualEntity implements IEntity {
}
}
private EntityType getEntityType(boolean isHint) {
return isHint ? EntityType.HINT : EntityType.ENTITY;
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -307,7 +307,7 @@ public final class MigrationEntity {
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.getType())
.type(entity.type())
.section(redactionLogEntry.getSection())
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java
View File

@ -128,19 +128,19 @@ public class Dictionary {
public void recommendEverywhere(TextEntity textEntity) {
addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
}
public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
}
public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
}

49
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
@ -33,13 +34,15 @@ public class TextEntity implements IEntity {
// primary key
@EqualsAndHashCode.Include
final TextRange textRange;
@EqualsAndHashCode.Include
final String type;
@EqualsAndHashCode.Include
final EntityType entityType;
final String id;
// primary key end
TextRange textRange;
@Builder.Default
List<TextRange> duplicateTextRanges = new ArrayList<>();
String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
final EntityType entityType;
@Builder.Default
final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
final ManualChangeOverwrite manualOverwrite;
@ -62,12 +65,30 @@ public class TextEntity implements IEntity {
SemanticNode deepestFullyContainingNode;
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) {
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
}
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
}
private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
}
public void addTextRange(TextRange textRange) {
duplicateTextRanges.add(textRange);
}
public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
return intersectingNodes.stream().anyMatch(clazz::isInstance);
@ -82,13 +103,13 @@ public class TextEntity implements IEntity {
public boolean isType(String type) {
return this.type.equals(type);
return type().equals(type);
}
public boolean isAnyType(List<String> types) {
return types.contains(type);
return types.contains(type());
}
@ -125,7 +146,6 @@ public class TextEntity implements IEntity {
.min(Comparator.comparingInt(Page::getNumber))
.orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
}
return positionsOnPagePerPage;
@ -193,7 +213,7 @@ public class TextEntity implements IEntity {
});
sb.delete(sb.length() - 2, sb.length());
sb.append("], type = \"");
sb.append(type);
sb.append(type());
sb.append("\", EntityType.");
sb.append(entityType);
sb.append("]");
@ -207,4 +227,11 @@ public class TextEntity implements IEntity {
return getManualOverwrite().getType().orElse(type);
}
@Override
public String value() {
return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
}
}

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java
View File

@ -19,6 +19,7 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -27,8 +28,12 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Document implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId = Collections.emptyList();
Set<Page> pages;
DocumentTree documentTree;
Integer numberOfPages;
@ -67,13 +72,6 @@ public class Document implements GenericSemanticNode {
}
@Override
public List<Integer> getTreeId() {
return Collections.emptyList();
}
@Override
public void setTreeId(List<Integer> tocId) {

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java
View File

@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Footer implements GenericSemanticNode {
@Builder.Default
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java
View File

@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Header implements GenericSemanticNode {
@Builder.Default
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java
View File

@ -24,20 +24,19 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Headline implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
SectionIdentifier sectionIdentifier;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java
View File

@ -32,8 +32,10 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Image implements GenericSemanticNode, IEntity {
@EqualsAndHashCode.Include
List<Integer> treeId;
String id;

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java
View File

@ -23,26 +23,23 @@ import lombok.experimental.FieldDefaults;
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Page {
@EqualsAndHashCode.Include
Integer number;
Integer height;
Integer width;
Integer rotation;
@EqualsAndHashCode.Exclude
List<SemanticNode> mainBody;
@EqualsAndHashCode.Exclude
Header header;
@EqualsAndHashCode.Exclude
Footer footer;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@Builder.Default
@EqualsAndHashCode.Exclude
Set<Image> images = new HashSet<>();
@ -57,19 +54,4 @@ public class Page {
return String.valueOf(number);
}
@Override
public int hashCode() {
return number;
}
@Override
public boolean equals(Object o) {
return o instanceof Page && o.hashCode() == this.hashCode();
}
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java
View File

@ -21,19 +21,18 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Paragraph implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java
View File

@ -24,19 +24,17 @@ import lombok.extern.slf4j.Slf4j;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Section implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock textBlock;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;

48
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
@ -80,8 +81,8 @@ public interface SemanticNode {
*/
default Set<Page> getPages(TextRange textRange) {
if (!getTextRange().contains(textRange)) {
throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange()));
if (!getTextRange().intersects(textRange)) {
throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
}
return getTextBlock().getPages(textRange);
}
@ -247,7 +248,7 @@ public interface SemanticNode {
*/
default boolean hasEntitiesOfType(String type) {
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
}
@ -260,7 +261,7 @@ public interface SemanticNode {
*/
default boolean hasEntitiesOfAnyType(String... types) {
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
}
@ -273,7 +274,7 @@ public interface SemanticNode {
*/
default boolean hasEntitiesOfAllTypes(String... types) {
return getEntities().stream().filter(TextEntity::active).map(TextEntity::getType).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
}
@ -286,7 +287,7 @@ public interface SemanticNode {
*/
default List<TextEntity> getEntitiesOfType(String type) {
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
}
@ -627,6 +628,27 @@ public interface SemanticNode {
}
/**
* For a given TextRange this function returns a List of rectangle around the text in the range.
* These Rectangles are split either by a new line or by a large gap in the current line.
* This is mainly used to find the positions of TextEntities
*
* @param textRange A TextRange to calculate the positions for.
* @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
*/
default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
if (isLeaf()) {
return getTextBlock().getPositionsPerPage(textRange);
}
Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
if (containingChildNode.isEmpty()) {
return getTextBlock().getPositionsPerPage(textRange);
}
return containingChildNode.get().getPositionsPerPage(textRange);
}
/**
* If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
* If called on the Document, it will return the cropbox of each page
@ -693,4 +715,18 @@ public interface SemanticNode {
return bBoxPerPage;
}
/**
* Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
* The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
*
* @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
* @see NodeVisitor
*/
default void accept(NodeVisitor visitor) {
visitor.visit(this);
streamChildren().forEach(childNode -> childNode.accept(visitor));
}
}

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java
View File

@ -28,8 +28,10 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Table implements SemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
DocumentTree documentTree;
@ -39,10 +41,8 @@ public class Table implements SemanticNode {
TextBlock textBlock;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();
@EqualsAndHashCode.Exclude
Map<Page, Rectangle2D> bBoxCache;
@ -128,7 +128,7 @@ public class Table implements SemanticNode {
List<Integer> rowsWithEntityOfType = getEntities().stream()
.filter(TextEntity::active)
.filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
.filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.type())))
.map(TextEntity::getIntersectingNodes)
.filter(node -> node instanceof TableCell)
.map(node -> (TableCell) node)
@ -153,7 +153,7 @@ public class Table implements SemanticNode {
.filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
.flatMap(Collection::stream)
.filter(TextEntity::active)
.noneMatch(entity -> types.contains(entity.getType())))
.noneMatch(entity -> types.contains(entity.type())))
.flatMap(this::streamRow)
.map(TableCell::getEntities)
.flatMap(Collection::stream);
@ -196,7 +196,7 @@ public class Table implements SemanticNode {
*/
public Stream<TableCell> streamTableCellsWhichContainType(String type) {
return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.type().equals(type)));
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java
View File

@ -23,8 +23,10 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class TableCell implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
int row;
int col;
@ -36,11 +38,9 @@ public class TableCell implements GenericSemanticNode {
TextBlock textBlock;
@EqualsAndHashCode.Exclude
DocumentTree documentTree;
@Builder.Default
@EqualsAndHashCode.Exclude
Set<TextEntity> entities = new HashSet<>();

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java
View File

@ -71,7 +71,16 @@ public class ConcatenatedTextBlock implements TextBlock {
private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList();
List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
if (atomicTextBlock.getTextRange().start() > textRange.end()) {
break; // early stop, following TextBlocks will never intersect
}
if (atomicTextBlock.getTextRange().intersects(textRange)) {
intersectingAtomicTextBlocks.add(atomicTextBlock);
}
}
return intersectingAtomicTextBlocks;
}
@ -122,13 +131,13 @@ public class ConcatenatedTextBlock implements TextBlock {
}
@Override
public Rectangle2D getPosition(int stringIdx) {
return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
}
public TextRange getLineTextRange(int lineNumber) {
if (atomicTextBlocks.size() == 1) {
@ -144,10 +153,10 @@ public class ConcatenatedTextBlock implements TextBlock {
return new TextRange(textRange.start(), textRange.start());
}
@Override
public List<Rectangle2D> getPositions(TextRange stringTextRange) {
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
if (textBlocks.isEmpty()) {

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -142,44 +142,39 @@ public class EntityLogCreatorService {
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<EntityLogEntry> entries = new ArrayList<>();
Set<String> processedIds = new HashSet<>();
document.getEntities()
.stream()
.filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity)));
return entries;
}
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
// Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
if (processedIds.contains(positionOnPage.getId())) {
continue;
}
processedIds.add(positionOnPage.getId());
EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
entityLogEntries.setId(positionOnPage.getId());
EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
.toList();
entityLogEntries.setPositions(rectanglesPerLine);
redactionLogEntities.add(entityLogEntries);
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
entityLogEntry.setId(positionOnPage.getId());
entityLogEntry.setPositions(rectanglesPerLine);
entityLogEntries.add(entityLogEntry);
}
return redactionLogEntities;
return entityLogEntries;
}
@ -190,8 +185,6 @@ public class EntityLogCreatorService {
return EntityLogEntry.builder()
.id(image.getId())
.value(image.value())
.color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.value(image.value())
.type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis())
@ -210,13 +203,12 @@ public class EntityLogCreatorService {
}
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType());
return EntityLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
@ -246,17 +238,16 @@ public class EntityLogCreatorService {
}
private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
private EntityLogEntry createEntityLogEntry(TextEntity entity) {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
boolean isHint = isHint(entity.getEntityType());
return EntityLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.getType())
.type(entity.type())
.section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
@ -282,16 +273,6 @@ public class EntityLogCreatorService {
}
private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
if (!isApplied && !isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) {

154
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -1,54 +1,70 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualChangesApplicationService {
EntityCreationService entityCreationService;
static double MATCH_THRESHOLD = 10;
public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
EntityFindingUtility entityFindingUtility;
if (IEntityToBeReCategorized instanceof Image image) {
@Deprecated
public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
entityToBeReCategorized.getMatchedRuleList().clear();
entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
if (entityToBeReCategorized instanceof Image image) {
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
return;
}
// need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
textEntity.remove("FINAL.0.0", "removed by manual recategorization");
if (entityToBeReCategorized instanceof TextEntity textEntity) {
textEntity.setType(manualRecategorization.getType());
}
}
public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction);
}
@Deprecated
public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
@ -57,96 +73,62 @@ public class ManualChangesApplicationService {
.findFirst()
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
.stream()
.map(ManualChangesApplicationService::toRectangle2D)
.collect(Collectors.toList()));
String value = manualResizeRedaction.getValue();
int newStartOffset = -1;
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction);
// Loop through nodes starting from the deepest fully containing node all the way to the document node
while (node != null) {
if (node.containsString(value)) {
SearchImplementation searchImplementation = new SearchImplementation(value, false);
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
for (TextRange textRange : textRanges) {
SemanticNode finalNode = node;
Map<String, List<TextEntity>> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity));
Optional<TextEntity> closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD);
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
.stream()
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
.collect(Collectors.toList());
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
// Remove all temp entities from the graph
tempEntities.forEach(TextEntity::removeFromGraph);
}
break;
if (closestEntity.isPresent()) {
copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
return;
}
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
if (node.hasParent()) {
node = node.getParent();
} else {
break;
}
}
}
if (closestEntity.getTextRange() != null) {
newStartOffset = closestEntity.getTextRange().start();
}
// need to reinsert the entity, due to the boundary having changed.
if (newStartOffset > -1) {
removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
}
private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) {
Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
entityToBeResized.setTextRange(closestEntity.getTextRange());
entityToBeResized.setTextAfter(closestEntity.getTextAfter());
entityToBeResized.setTextBefore(closestEntity.getTextBefore());
entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
entityToBeResized.setValue(closestEntity.getValue());
entityToBeResized.setPages(newIntersectingPages);
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
}
public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
TextRange textRange,
List<TextEntity> tempEntities,
ClosestEntity closestEntity) {
double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
if (currentDistance < closestEntity.getDistance()) {
closestEntity.setDistance(currentDistance);
closestEntity.setTextRange(textRange);
}
}
private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
double difference = 0;
difference += position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
difference += position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
difference += position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
difference += position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
return difference;
}
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
entityToBeResized.setIntersectingNodes(new LinkedList<>());
entityToBeResized.setDeepestFullyContainingNode(null);
entityToBeResized.setPages(new HashSet<>());
entityToBeResized.getTextRange().setStart(newStartOffset);
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
}
public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -100,11 +100,11 @@ public class RedactionLogCreatorService {
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getEntityType());
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.color(getColor(entity.type(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.getType())
.type(entity.type())
.redacted(entity.applied())
.isHint(isHint)
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))

120
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java
View File

@ -3,10 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -23,11 +21,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@ -62,6 +57,8 @@ public class UnprocessedChangesService {
final EntityEnrichmentService entityEnrichmentService;
final ManualEntityCreationService manualEntityCreationService;
final DictionaryService dictionaryService;
final ManualChangesApplicationService manualChangesApplicationService;
EntityCreationService entityCreationService;
@ -145,8 +142,21 @@ public class UnprocessedChangesService {
continue;
}
TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get());
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId()))
.findFirst();
if (optionalManualResizeRedaction.isPresent()) {
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction);
// If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
// from the graph, so it does not get processed and sent back to persistence-service to update its value.
if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) {
correctEntity.removeFromGraph();
}
}
}
// remove all temp entities from the graph
@ -154,101 +164,27 @@ public class UnprocessedChangesService {
}
public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
if (optionalManualResizeRedaction.isEmpty()) {
return;
}
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
.stream()
.filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
.findFirst()
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
correctEntity.setValue(closestEntity.getValue());
correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore());
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
String value = manualResizeRedaction.getValue();
int newStartOffset = -1;
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
// Loop through nodes starting from the deepest fully containing node all the way to the document node
while (node != null) {
if (node.containsString(value)) {
SearchImplementation searchImplementation = new SearchImplementation(value, false);
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
for (TextRange textRange : textRanges) {
SemanticNode finalNode = node;
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
.stream()
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
.collect(Collectors.toList());
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
// Remove all temp entities from the graph
tempEntities.forEach(TextEntity::removeFromGraph);
}
break;
}
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
if (node.hasParent()) {
node = node.getParent();
} else {
break;
}
}
if (closestEntity.getTextRange() != null) {
newStartOffset = closestEntity.getTextRange().start();
}
// need to reinsert the entity, due to the boundary having changed.
removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
}
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
entityToBeResized.removeFromGraph();
entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
entityToBeResized.setIntersectingNodes(new LinkedList<>());
entityToBeResized.setDeepestFullyContainingNode(null);
entityToBeResized.setPages(new HashSet<>());
entityToBeResized.getTextRange().setStart(newStartOffset);
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
// Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
if (newStartOffset > -1) {
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
}
}
private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
}
correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
return correctEntity;
}

207
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Collection;
@ -18,24 +19,21 @@ import org.kie.api.runtime.KieSession;
import com.google.common.base.Functions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import lombok.RequiredArgsConstructor;
@ -203,6 +201,7 @@ public class EntityCreationService {
return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
}
public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
checkIfBothStartAndEndAreEmpty(starts, stops);
@ -233,9 +232,10 @@ public class EntityCreationService {
return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
}
public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0);
return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node, 0);
}
@ -282,22 +282,11 @@ public class EntityCreationService {
}
private void checkIfBothStartAndEndAreEmpty(String start, String end) {
checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
}
private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
throw new IllegalArgumentException("Start and end values are empty!");
}
}
public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream().filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
.stream()
.filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
.map(bounds -> byTextRange(bounds, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
@ -309,7 +298,9 @@ public class EntityCreationService {
TextBlock textBlock = node.getTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, false);
return searchImplementation.getBoundaries(textBlock, node.getTextRange())
.stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.stream()
.map(boundary -> toLineAfterTextRange(textBlock, boundary))
.filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.map(boundary -> byTextRange(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
@ -321,7 +312,9 @@ public class EntityCreationService {
TextBlock textBlock = node.getTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, true);
return searchImplementation.getBoundaries(textBlock, node.getTextRange())
.stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.stream()
.map(boundary -> toLineAfterTextRange(textBlock, boundary))
.filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.map(boundary -> byTextRange(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
@ -332,7 +325,9 @@ public class EntityCreationService {
TextBlock textBlock = node.getTextBlock();
return RedactionSearchUtility.findTextRangesByString(string, textBlock)
.stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.stream()
.map(boundary -> toLineAfterTextRange(textBlock, boundary))
.filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.map(boundary -> byTextRange(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
@ -343,7 +338,9 @@ public class EntityCreationService {
TextBlock textBlock = node.getTextBlock();
return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock)
.stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.stream()
.map(boundary -> toLineAfterTextRange(textBlock, boundary))
.filter(boundary -> isValidEntityTextRange(textBlock, boundary))
.map(boundary -> byTextRange(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
@ -352,7 +349,8 @@ public class EntityCreationService {
public Stream<TextEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
return tableNode.streamTableCells().flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
return tableNode.streamTableCells()
.flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
tableCell,
type,
entityType,
@ -545,7 +543,7 @@ public class EntityCreationService {
public Optional<TextEntity> byPrefixExpansionRegex(TextEntity entity, String regexPattern) {
int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
}
@ -553,18 +551,10 @@ public class EntityCreationService {
int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
}
private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
if (textBlock.getNextLinebreak(end) < expandedEnd) {
return end;
}
return expandedEnd;
}
/**
* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
* If the document already contains an equal redaction entity, then the original Entity is returned.
@ -581,6 +571,7 @@ public class EntityCreationService {
return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE));
}
/**
* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
* If the document already contains an equal redaction entity, then the original Entity is returned.
@ -599,9 +590,21 @@ public class EntityCreationService {
throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
}
TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
if (node.getEntities().contains(entity)) {
return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();
Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
if (optionalTextEntity.isEmpty()) {
return optionalTextEntity; // Entity has been recategorized and should not be created at all.
}
TextEntity existingEntity = optionalTextEntity.get();
if (existingEntity.getTextRange().equals(textRange)) {
return optionalTextEntity; // exactly the same entity, return directly
}
if (!existingEntity.resized()) {
addDuplicateEntityToGraph(existingEntity, textRange, node);
return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
}
return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
}
addEntityToGraph(entity, node);
entity.addEngines(engines);
@ -610,10 +613,23 @@ public class EntityCreationService {
}
// Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
@Deprecated(forRemoval = true)
public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
addEntityToGraph(entity, node);
return entity;
}
// Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
@Deprecated(forRemoval = true)
public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node, String id) {
TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, id);
addEntityToGraph(entity, node);
return entity;
}
@ -631,7 +647,7 @@ public class EntityCreationService {
return entitiesToMerge.get(0);
}
TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType);
TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node);
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
entitiesToMerge.stream()
@ -662,6 +678,7 @@ public class EntityCreationService {
return newEntity;
}
public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
@ -683,7 +700,8 @@ public class EntityCreationService {
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
"No entity present!"));
}
@ -693,6 +711,18 @@ public class EntityCreationService {
}
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
}
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
}
public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
@ -702,21 +732,6 @@ public class EntityCreationService {
}
public TextEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
TextEntity highlightEntity = TextEntity.initialEntityNode(new TextRange(tableCell.getTextRange().start(), tableCell.getTextRange().start()), type, entityType);
String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
highlightEntity.setPositionsOnPagePerPage(tableCell.getBBox()
.entrySet()
.stream()
.map(entry -> new PositionOnPage(positionId, entry.getKey(), List.of(entry.getValue())))
.toList());
addEntityToGraph(highlightEntity, tableCell);
return highlightEntity;
}
public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) {
return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange);
@ -727,9 +742,17 @@ public class EntityCreationService {
DocumentTree documentTree = node.getDocumentTree();
try {
addEntityToGraph(entity, documentTree);
entity.addIntersectingNode(documentTree.getRoot().getNode());
documentTree.getRoot().getNode().getEntities().add(entity);
if (node.getEntities().contains(entity)) {
// If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
node.getEntities().stream()//
.filter(e -> e.equals(entity))//
.filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
.findAny()//
.ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
} else {
entity.addIntersectingNode(documentTree.getRoot().getNode());
addEntityToGraph(entity, documentTree);
}
} catch (NoSuchElementException e) {
entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
@ -740,6 +763,33 @@ public class EntityCreationService {
}
private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
entityToDuplicate.addTextRange(newTextRange);
SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
.stream()
.sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
.filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
.findFirst()
.orElse(node.getDocumentTree().getRoot().getNode());
entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode);
Set<SemanticNode> additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange);
additionalIntersectingNodes.forEach(additionalIntersectingNode -> {
if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) {
return;
}
additionalIntersectingNode.getEntities().add(entityToDuplicate);
additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate));
entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
});
}
private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
@ -757,45 +807,4 @@ public class EntityCreationService {
}
private static void addToPages(TextEntity entity) {
Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
entity.getPages().addAll(pages);
pages.forEach(page -> page.getEntities().add(entity));
}
private static void addEntityToNodeEntitySets(TextEntity entity) {
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
}
private static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) {
return true;
}
TextEntity previousEntity = entitiesToMerge.get(0);
for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
boolean typeMatches = textEntity.getType().equals(previousEntity.getType());
boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
boolean intersects = textEntity.intersects(previousEntity);
if (!typeMatches || !entityTypeMatches || !intersects) {
return false;
}
}
return true;
}
private static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
if (textBlock.getTextRange().end() == textRange.end()) {
return new TextRange(textRange.end(), textRange.end());
}
return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
}
}

91
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java Normal file
View File

@ -0,0 +1,91 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
public class EntityCreationUtility {
public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
}
public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
throw new IllegalArgumentException("Start and end values are empty!");
}
}
public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
if (textBlock.getNextLinebreak(end) < expandedEnd) {
return end;
}
return expandedEnd;
}
public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
if (initialIntersectingNode.getTextRange().intersects(textRange)) {
initialIntersectingNode.accept(visitor);
}
return visitor.getIntersectingNodes();
}
public static void addToPages(TextEntity entity) {
Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
entity.getPages().addAll(pages);
pages.forEach(page -> page.getEntities().add(entity));
}
public static void addEntityToNodeEntitySets(TextEntity entity) {
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
}
public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) {
return true;
}
TextEntity previousEntity = entitiesToMerge.get(0);
for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
boolean typeMatches = textEntity.type().equals(previousEntity.type());
boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
boolean intersects = textEntity.intersects(previousEntity);
if (!typeMatches || !entityTypeMatches || !intersects) {
return false;
}
}
return true;
}
public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
if (textBlock.getTextRange().end() == textRange.end()) {
return new TextRange(textRange.end(), textRange.end());
}
return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
}
}

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java
View File

@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy;
import java.awt.geom.Rectangle2D;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
@ -17,6 +18,7 @@ import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
@ -58,28 +60,28 @@ public class EntityFindingUtility {
return Optional.empty();
}
Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
.filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition()))
.min(Comparator.comparingDouble(entity -> calculateMinDistance(manualEntity.getEntityPosition(), entity)));
.map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build())
.min(Comparator.comparingDouble(ClosestEntity::getDistance));
if (optionalClosestEntity.isEmpty()) {
log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition());
return Optional.empty();
}
TextEntity closestEntity = optionalClosestEntity.get();
double distance = calculateMinDistance(manualEntity.getEntityPosition(), closestEntity);
if (distance > matchThreshold) {
ClosestEntity closestEntity = optionalClosestEntity.get();
if (closestEntity.getDistance() > matchThreshold) {
log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
manualEntity.getValue(),
manualEntity.getEntityPosition().get(0).pageNumber(),
manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
distance,
closestEntity.getDistance(),
matchThreshold);
return Optional.empty();
}
return Optional.of(closestEntity);
return Optional.of(closestEntity.getTextEntity());
}
@ -97,7 +99,7 @@ public class EntityFindingUtility {
}
private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
public static double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
if (originalPositions.size() != countRectangles(entity)) {
return Double.MAX_VALUE;
@ -115,7 +117,7 @@ public class EntityFindingUtility {
}
private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
return entity.getPositionsOnPagePerPage()
.stream()
@ -128,7 +130,7 @@ public class EntityFindingUtility {
}
public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
// mirrored coordinates safe comparison
double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
@ -163,7 +165,10 @@ public class EntityFindingUtility {
return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream()
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
.filter(Optional::isPresent)
.map(Optional::get)
.distinct()
.collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java Normal file
View File

@ -0,0 +1,31 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import lombok.Getter;
public class IntersectingNodeVisitor implements NodeVisitor {
@Getter
private Set<SemanticNode> intersectingNodes;
private final TextRange textRange;
public IntersectingNodeVisitor(TextRange textRange) {
this.textRange = textRange;
this.intersectingNodes = new HashSet<>();
}
@Override
public void visit(SemanticNode node) {
if (node.getTextRange().intersects(textRange)) {
intersectingNodes.add(node);
}
}
}

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -46,17 +47,24 @@ public class ManualEntityCreationService {
}
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions,
SemanticNode node,
String dossierTemplateId) {
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd().stream()
.filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) &&
manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate())))
List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> !(idRemovals.stream()
.map(BaseAnnotation::getAnnotationId)
.toList()
.contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
.isBefore(idRemovals.stream()
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.findFirst()
.get()
.getRequestDate())))
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> {
if (manualEntity.getEntityType().equals(EntityType.HINT)) {
manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
} else {
@ -75,12 +83,12 @@ public class ManualEntityCreationService {
List<ManualEntity> notFoundManualEntities = new LinkedList<>();
for (ManualEntity manualEntity : manualEntities) {
Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
if (optionalRedactionEntity.isEmpty()) {
Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
if (optionalClosestEntity.isEmpty()) {
notFoundManualEntities.add(manualEntity);
continue;
}
createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange());
createCorrectEntity(manualEntity, optionalClosestEntity.get());
}
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
return notFoundManualEntities;
@ -90,25 +98,29 @@ public class ManualEntityCreationService {
/**
* Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
*
* @param manualEntity The entity identifier for the RedactionEntity.
* @param node The SemanticNode associated with the RedactionEntity.
* @param closestTextRange The closest Boundary to the RedactionEntity.
* @param manualEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity.
*/
private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
correctEntity.setValue(closestEntity.getValue());
correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore());
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
// AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
}
correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java
View File

@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor;

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java Normal file
View File

@ -0,0 +1,8 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
public interface NodeVisitor {
void visit(SemanticNode node);
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
View File

@ -42,7 +42,7 @@ public class EntityDroolsExecutionService {
EntityEnrichmentService entityEnrichmentService;
ObservationRegistry observationRegistry;
ManualChangesApplicationService manualChangesApplicationService;
RedactionServiceSettings settings;
@ -73,7 +73,6 @@ public class EntityDroolsExecutionService {
KieSession kieSession = kieContainer.newKieSession();
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
@ -88,7 +87,7 @@ public class EntityDroolsExecutionService {
fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
if (manualRedactions != null) {
manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert);
manualRedactions.getResizeRedactions().forEach(kieSession::insert);
manualRedactions.getRecategorizations().forEach(kieSession::insert);
manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
manualRedactions.getForceRedactions().forEach(kieSession::insert);

32
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -293,7 +293,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
@ -1331,6 +1331,36 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
}
@Test
@SneakyThrows
public void testResizeWithUpdateDictionaryTrue() {
String pdfFile = "files/new/crafted document.pdf";
AnalyzeRequest request = uploadFileToStorage(pdfFile);
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
analyzeService.analyze(request);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get();
request.setManualRedactions(ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.updateDictionary(true)
.annotationId(david.getId())
.requestDate(OffsetDateTime.now())
.value("David Ksenia")
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build()))
.addToAllDossiers(false)
.build()))
.build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get();
assertEquals(resizedEntity.getState(), EntryState.APPLIED);
assertEquals(resizedEntity.getValue(), "David Ksenia");
}
private IdRemoval getIdRemoval(String id) {

13
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -5,16 +5,18 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
public class TextEntityTest {
@Test
public void testMatchedRule() {
TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
ManualEntity entity = ManualEntity.builder()
.type("PII")
.entityType(EntityType.ENTITY)
.build();
entity.skip("CBI.1.0", "");
entity.skip("CBI.2.0", "");
entity.skip("CBI.3.0", "");
@ -29,7 +31,10 @@ public class TextEntityTest {
@Test
public void testMatchedRuleWithNonsense() {
TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
ManualEntity entity = ManualEntity.builder()
.type("PII")
.entityType(EntityType.ENTITY)
.build();
assertThrows(IllegalArgumentException.class, () -> {
entity.skip("", "");
});

21
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java
View File

@ -49,6 +49,21 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
}
@Test
public void assertEntitiesAreDuplicatedWithTheirTableCell() {
Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf");
List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList();
assertEquals(3, entities.size());
assertEquals(1, entities.stream().distinct().count());
assertEquals(2, entities.get(0).getDuplicateTextRanges().size());
var node = entities.get(0).getDeepestFullyContainingNode();
assertTrue(node.getTextRange().contains(entities.get(0).getTextRange()));
assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr)));
}
@Test
public void assertCollectAllEntitiesWorks() {
@ -82,7 +97,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assert start != -1;
TextRange textRange = new TextRange(start, start + searchTerm.length());
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
entityCreationService.addEntityToGraph(textEntity, document);
return textEntity;
}
@ -241,7 +256,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assert start != -1;
TextRange textRange = new TextRange(start, start + searchTerm.length());
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
entityCreationService.addEntityToGraph(textEntity, document);
assertEquals("2.6.1 Summary of ", textEntity.getTextBefore());
@ -299,7 +314,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assert start != -1;
TextRange textRange = new TextRange(start, start + searchTerm.length());
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
entityCreationService.addEntityToGraph(textEntity, document);
Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();

25
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static org.mockito.Mockito.doThrow;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
import java.awt.Color;
@ -34,6 +34,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@ -41,16 +45,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ -245,8 +244,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
System.out.printf("%d Searches took %s s, average %.2f ms\n", numberOfRuns, ((float) totalSearchTime / 1000), totalSearchTime / numberOfRuns);
System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns);
System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size());
assert document.getEntities().size() == foundEntities.size();
for (TextEntity entity : document.getEntities()) {
var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get();
assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange()));
}
assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size();
assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size();
drawAllEntities(filename, document);
}
@ -300,7 +303,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
searchImplementation.getBoundaries(textBlock, textBlock.getTextRange())
.stream()
.filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
.map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType))
.map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document))
.forEach(foundEntities::add);
}

83
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -4,11 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertTrue;
import java.awt.geom.Rectangle2D;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.OffsetDateTime;
import java.util.Comparator;
@ -121,6 +123,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
@ -147,7 +150,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String filePath = "files/new/crafted document.pdf";
AnalyzeRequest request = uploadFileToStorage(filePath);
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
analyzeService.analyze(request);
String testEntityValue1 = "Desiree";
String testEntityValue2 = "Melanie";
@ -157,7 +160,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
String idToResize = redactionLog.getEntityLogEntry()
.stream()
@ -165,24 +168,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
.get()
.getId();
List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
.get(0)
.getRectanglePerLine()
.stream()
.map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3))
.toList();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(idToResize)
.value(expandedEntityKeyword)
.positions(resizedPositions)
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.build();
manualResizeRedaction.setUpdateDictionary(false);
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
.annotationId(idToResize)
.value(expandedEntityKeyword)
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
.addToAllDossiers(false)
.updateDictionary(false)
.requestDate(OffsetDateTime.now())
.build());
request.setManualRedactions(manualRedactions);
AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
analyzeService.reanalyze(request);
redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
@ -225,7 +222,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.build()));
.build()));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
@ -248,7 +245,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.requestDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build())));
analyzeService.reanalyze(request);
@ -267,6 +264,45 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
}
@Test
public void testManualRedactionIsDuplicatedWithTableCells() throws IOException {
System.out.println("testManualRedaction");
long start = System.currentTimeMillis();
String pdfFile = "files/Minimal Examples/Meto1_Page22.pdf";
ManualRedactions manualRedactions = new ManualRedactions();
String manualAddId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setType("name");
manualRedactionEntry.setValue("Surface Water");
manualRedactionEntry.setReason("Dictionary Request");
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(76.584f).topLeftY(506.56238f).width(57.4329f).height(9.065156f).page(1).build()));
manualRedactions.setEntriesToAdd(Set.of(manualRedactionEntry));
AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions);
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/" + Path.of(pdfFile).getFileName() + "MANUAL_REDACTION_TEST.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
long end = System.currentTimeMillis();
var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
assertTrue(optionalEntry.isPresent());
assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
System.out.println("duration: " + (end - start));
System.out.println("numberOfPages: " + result.getNumberOfPages());
}
@Test
public void testReCategorizeToVertebrateChangesCbiAuthor() {
@ -296,7 +332,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
ManualRecategorization recategorization = ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.type("vertebrate")
.type("vertebrate")
.annotationId(oxfordUniversityPress.getId())
.fileId(TEST_FILE_ID)
.build();
@ -345,18 +381,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())
.type("manual")
.value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
.positions(List.of(//
new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
.build()));
.build()));
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())
.value("Expand to Hint")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
.updateDictionary(false)
.updateDictionary(false)
.build();
manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
request.setManualRedactions(manualRedactions);

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java
View File

@ -7,12 +7,14 @@ import static org.wildfly.common.Assert.assertFalse;
import java.awt.geom.Rectangle2D;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -50,7 +52,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertEquals(biggerEntity.getTextRange(), entity.getTextRange());
assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
assertEquals(biggerEntity.getIntersectingNodes(), entity.getIntersectingNodes());
assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
assertEquals(biggerEntity.getPages(), entity.getPages());
assertEquals(biggerEntity.getValue(), entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java
View File

@ -119,7 +119,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
entity.getManualOverwrite().addChange(imageRecategorizationRequest);
assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
assertTrue(entity.getManualOverwrite().getRecategorized().get());
assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.getType()));
assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -167,14 +167,14 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<TextEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type)));
return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type)));
}
private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<TextEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type))));
return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type))));
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java
View File

@ -28,6 +28,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
@Autowired
protected EntityEnrichmentService entityEnrichmentService;
@Autowired
protected ManualChangesApplicationService manualChangesApplicationService;
protected EntityCreationService entityCreationService;
protected KieSession kieSession;
@ -72,7 +74,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
kieSession = kieContainer.newKieSession();
entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("entityCreationService", entityCreationService);
}

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -174,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
analyzeService.reanalyze(request);
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build());
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).page(1).build());
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL");
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@ -221,7 +221,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
analyzeService.analyze(request);
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
List<Rectangle> positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build());
List<Rectangle> positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build());
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
@ -285,7 +285,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
analyzeService.analyze(request);
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@ -323,7 +323,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
analyzeService.analyze(request);
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build());
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).page(1).build());
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL");
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@ -361,7 +361,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
analyzeService.analyze(request);
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist");
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());

67
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -84,7 +84,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -92,7 +92,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -102,7 +102,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
end
@ -110,7 +110,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -119,7 +119,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
// Rule unit: CBI.2
rule "CBI.2.0: Do not redact genitive CBI Author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
$entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -148,7 +148,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
$table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
$cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
$tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
$authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
$authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
then
$authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
end
@ -301,7 +301,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -309,7 +309,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -625,7 +625,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -634,7 +634,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@ -693,7 +693,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@ -709,7 +709,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -733,7 +733,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -811,13 +811,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -825,7 +824,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -883,8 +882,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -895,8 +894,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -911,8 +910,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -924,8 +923,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -936,8 +935,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -959,8 +958,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -971,8 +970,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -982,8 +981,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

93
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
end
@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
// Rule unit: CBI.2
rule "CBI.2.0: Do not redact genitive CBI Author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
$entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
$table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
$cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
$tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
$authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
$authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
then
$authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
end
@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
rule "CBI.13.0: Ignore CBI Address recommendations"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
$entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION)
then
$entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
retract($entity)
@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
$sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
$sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
rule "CBI.18.0: Expand CBI_author entities with firstname initials"
no-loop true
when
$entityToExpand: TextEntity(type == "CBI_author",
$entityToExpand: TextEntity(type() == "CBI_author",
value.matches("[^\\s]+"),
textAfter.startsWith(" "),
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
// Rule unit: CBI.19
rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
when
$entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
@ -668,7 +668,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
then
entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
@ -680,7 +680,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
then
entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
@ -707,7 +707,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -715,7 +715,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -1047,7 +1047,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1134,21 +1134,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
// Rule unit: ETC.4
rule "ETC.4.0: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.4.1: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
end
rule "ETC.4.2: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)");
end
@ -1158,7 +1158,7 @@ rule "ETC.4.2: Redact dossier dictionary entries"
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@ -1212,7 +1212,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
rule "ETC.9.0: Redact skipped impurities"
when
FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
$skippedImpurities: TextEntity(type == "skipped_impurities")
$skippedImpurities: TextEntity(type() == "skipped_impurities")
then
$skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
@ -1220,7 +1220,7 @@ rule "ETC.9.0: Redact skipped impurities"
rule "ETC.9.1: Redact impurities"
when
FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
$skippedImpurities: TextEntity(type == "impurities")
$skippedImpurities: TextEntity(type() == "impurities")
then
$skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
@ -1229,7 +1229,7 @@ rule "ETC.9.1: Redact impurities"
// Rule unit: ETC.10
rule "ETC.10.0: Redact Product Composition Information"
when
$compositionInformation: TextEntity(type == "product_composition")
$compositionInformation: TextEntity(type() == "product_composition")
then
$compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
end
@ -1256,7 +1256,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -1278,7 +1278,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author"
then
nerEntities.getNerEntityList().stream()
.filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
end
@ -1289,7 +1289,7 @@ rule "AI.3.0: Recommend authors from AI as PII"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
end
@ -1303,7 +1303,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -1381,13 +1381,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -1395,7 +1394,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -1453,8 +1452,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -1465,8 +1464,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -1481,8 +1480,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1494,8 +1493,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -1506,8 +1505,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -1529,8 +1528,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -1541,8 +1540,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -1552,8 +1551,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

37
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -466,7 +466,9 @@ rule "DOC.7.1: Performing Laboratory (Country)"
then
nerEntities.streamEntitiesOfType("COUNTRY")
.filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
.map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
.flatMap(Optional::stream)
.collect(Collectors.toList())
.forEach(entity -> {
entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
});
@ -475,7 +477,7 @@ rule "DOC.7.1: Performing Laboratory (Country)"
rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
when
$section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
$countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
$countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
then
$countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
end
@ -1161,7 +1163,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -1239,13 +1241,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -1253,7 +1254,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -1311,8 +1312,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -1323,8 +1324,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1336,8 +1337,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -1348,8 +1349,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -1371,8 +1372,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -75,7 +75,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -153,13 +153,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -167,7 +166,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -225,8 +224,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);

63
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -339,7 +339,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
$sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
$sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -442,7 +442,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
rule "CBI.18.0: Expand CBI_author entities with firstname initials"
no-loop true
when
$entityToExpand: TextEntity(type == "CBI_author",
$entityToExpand: TextEntity(type() == "CBI_author",
value.matches("[^\\s]+"),
textAfter.startsWith(" "),
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@ -460,7 +460,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
// Rule unit: CBI.19
rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
when
$entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
@ -505,7 +505,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -513,7 +513,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -733,7 +733,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -742,7 +742,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@ -800,7 +800,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
// Rule unit: ETC.4
rule "ETC.4.0: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -810,7 +810,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@ -869,7 +869,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -893,7 +893,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -971,13 +971,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -985,7 +984,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -1043,8 +1042,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -1055,8 +1054,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -1071,8 +1070,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1084,8 +1083,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -1096,8 +1095,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -1119,8 +1118,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -1131,8 +1130,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -1142,8 +1141,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

49
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -71,7 +71,7 @@ query "getFileAttributes"
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -83,7 +83,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -98,7 +98,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -112,7 +112,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -190,13 +190,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -204,7 +203,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -249,8 +248,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -261,8 +260,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -277,8 +276,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -290,8 +289,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -302,8 +301,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -325,8 +324,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -337,8 +336,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -348,8 +347,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

39
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -225,7 +225,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -303,13 +303,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -317,7 +316,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -375,8 +374,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -387,8 +386,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -400,8 +399,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -412,8 +411,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -435,8 +434,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -447,8 +446,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -458,8 +457,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

43
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -125,7 +125,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -203,13 +203,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -217,7 +216,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -275,8 +274,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -287,8 +286,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -303,8 +302,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -316,8 +315,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -328,8 +327,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -351,8 +350,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -363,8 +362,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -374,8 +373,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Meto1_Page22.pdf Normal file
View File

Binary file not shown.

49
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -81,7 +81,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -89,7 +89,7 @@ rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -99,7 +99,7 @@ rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
end
@ -107,7 +107,7 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -116,7 +116,7 @@ rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
// Rule unit: CBI.2
rule "CBI.2.0: Don't redact genitive CBI_author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
$entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -145,7 +145,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
$table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
$cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
$tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
$author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
$author: TextEntity(type() == "CBI_author", active()) from $tableCell.getEntities()
then
$author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
end
@ -298,7 +298,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -306,7 +306,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -452,7 +452,7 @@ rule "ETC.3.1: Redact logos (non vertebrate study)"
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
@ -469,7 +469,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -494,7 +494,7 @@ rule "MAN.0.0: Apply manual resize redaction"
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -575,8 +575,7 @@ rule "MAN.3.0: Apply entity recategorization"
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
update($entityToBeRecategorized);
end
rule "MAN.3.1: Apply image recategorization"
@ -617,8 +616,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -629,8 +628,8 @@ rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !resized(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -645,8 +644,8 @@ rule "X.1.0: merge intersecting Entities of same type"
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, entityType == EntityType.ENTITY, !resized(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -658,8 +657,8 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -670,8 +669,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
$entity: TextEntity($type: type(), entityType == EntityType.ENTITY, active())
$recommendation: TextEntity(intersects($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@ -695,8 +694,8 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
$higherRank: TextEntity($type: type(), entityType == EntityType.ENTITY, active())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");

99
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
end
@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
$entity: TextEntity(type() == "CBI_address", dictionaryEntry)
then
$entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
// Rule unit: CBI.2
rule "CBI.2.0: Do not redact genitive CBI Author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
$entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
$table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
$cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
$tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
$authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
$authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
then
$authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
end
@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
rule "CBI.13.0: Ignore CBI Address recommendations"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
$entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION)
then
$entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
retract($entity)
@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
$sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
$sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
rule "CBI.18.0: Expand CBI_author entities with firstname initials"
no-loop true
when
$entityToExpand: TextEntity(type == "CBI_author",
$entityToExpand: TextEntity(type() == "CBI_author",
value.matches("[^\\s]+"),
textAfter.startsWith(" "),
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
// Rule unit: CBI.19
rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
when
$entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
@ -667,7 +667,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
then
entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
@ -679,7 +679,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
then
entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
@ -706,7 +706,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -714,7 +714,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$pii: TextEntity(type == "PII", dictionaryEntry)
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -1045,7 +1045,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1056,7 +1056,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@ -1142,21 +1142,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
// Rule unit: ETC.4
rule "ETC.4.0: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.4.1: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
end
rule "ETC.4.2: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)");
end
@ -1166,7 +1166,7 @@ rule "ETC.4.2: Redact dossier dictionary entries"
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
when
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@ -1219,7 +1219,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
rule "ETC.9.0: Redact skipped impurities"
when
FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
$skippedImpurities: TextEntity(type == "skipped_impurities")
$skippedImpurities: TextEntity(type() == "skipped_impurities")
then
$skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
@ -1227,7 +1227,7 @@ rule "ETC.9.0: Redact skipped impurities"
rule "ETC.9.1: Redact impurities"
when
FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
$skippedImpurities: TextEntity(type == "impurities")
$skippedImpurities: TextEntity(type() == "impurities")
then
$skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
@ -1235,7 +1235,7 @@ rule "ETC.9.1: Redact impurities"
// Rule unit: ETC.10
rule "ETC.10.0: Redact Product Composition Information"
when
$compositionInformation: TextEntity(type == "product_composition")
$compositionInformation: TextEntity(type() == "product_composition")
then
$compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
end
@ -1256,7 +1256,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne
rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -1264,7 +1264,7 @@ rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)"
rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$dossierRedaction: TextEntity(type == "dossier_redaction")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -1278,7 +1278,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -1300,7 +1300,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author"
then
nerEntities.getNerEntityList().stream()
.filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
end
@ -1311,7 +1311,7 @@ rule "AI.3.0: Recommend authors from AI as PII"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
end
//------------------------------------ Manual redaction rules ------------------------------------
@ -1324,7 +1324,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -1402,13 +1402,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -1416,7 +1415,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -1474,8 +1473,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -1486,8 +1485,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
@ -1502,8 +1501,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1515,8 +1514,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -1527,8 +1526,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -1552,8 +1551,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);
@ -1564,8 +1563,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@ -1576,8 +1575,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

37
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -464,7 +464,9 @@ rule "DOC.7.1: Performing Laboratory (Country)"
then
nerEntities.streamEntitiesOfType("COUNTRY")
.filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
.map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
.flatMap(Optional::stream)
.collect(Collectors.toList())
.forEach(entity -> {
entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
});
@ -473,7 +475,7 @@ rule "DOC.7.1: Performing Laboratory (Country)"
rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
when
$section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
$countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
$countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
then
$countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
end
@ -1308,7 +1310,7 @@ rule "MAN.0.0: Apply manual resize redaction"
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@ -1386,13 +1388,12 @@ rule "MAN.3.0: Apply entity recategorization"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
update($entityToBeRecategorized);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
@ -1400,7 +1401,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
@ -1457,8 +1458,8 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
@ -1469,8 +1470,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@ -1482,8 +1483,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
@ -1494,8 +1495,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@ -1519,8 +1520,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
$entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
retract($recommendation);

2
redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt
View File

File diff suppressed because one or more lines are too long

2
redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt
View File

File diff suppressed because one or more lines are too long

2
redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt
View File

File diff suppressed because one or more lines are too long