Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10200: Spike: Performant update logic for facts in working memory

Browse Source
This commit is contained in:
maverickstuder 2024-11-29 15:06:35 +01:00
parent d34921acd8
commit 27b5a1afcd
51 changed files with 499 additions and 693 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java
View File

@ -9,6 +9,10 @@ import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
@ -17,6 +21,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.utils.EntityCreationUtility;
import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@ -358,6 +364,36 @@ public class DocumentTree {
}
public boolean addEntityToGraph(TextEntity entity) {
getRoot().getNode().addThisToEntityIfIntersects(entity);
TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
EntityEnrichmentService.enrichEntity(entity, textBlock);
EntityCreationUtility.addToPages(entity);
EntityCreationUtility.addEntityToNodeEntitySets(entity);
for (TextEntity textEntity : entity.getDeepestFullyContainingNode().getEntities()) {
if (entity.intersects(textEntity) && !entity.equals(textEntity)) {
if (textEntity.getTextRange().equals(entity.getTextRange())) {
textEntity.getRelations().put(textEntity, new Equality(entity, textEntity));
entity.getRelations().put(textEntity, new Equality(textEntity, entity));
} else if (textEntity.containedBy(entity)) {
entity.getRelations().put(textEntity, new Containment(entity, textEntity));
textEntity.getRelations().put(entity, new Intersection(textEntity, entity));
} else if (entity.containedBy(textEntity)) {
textEntity.getRelations().put(entity, new Containment(textEntity, entity));
entity.getRelations().put(textEntity, new Intersection(entity, textEntity));
} else {
entity.getRelations().put(textEntity, new Intersection(entity, textEntity));
textEntity.getRelations().put(entity, new Intersection(textEntity, entity));
}
}
}
return true;
}
@Builder
@Getter
@AllArgsConstructor

32
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java Normal file
View File

@ -0,0 +1,32 @@
package com.iqser.red.service.redaction.v1.server.model.document;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import lombok.Getter;
public class IntersectingNodeVisitor extends AbstractNodeVisitor {
@Getter
private Set<SemanticNode> intersectingNodes;
private final TextRange textRange;
public IntersectingNodeVisitor(TextRange textRange) {
this.textRange = textRange;
this.intersectingNodes = new HashSet<>();
}
@Override
public void visitNodeDefault(SemanticNode node) {
if (textRange.intersects(node.getTextRange())) {
intersectingNodes.add(node);
}
}
}

16
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java
View File

@ -1,16 +0,0 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Getter
@RequiredArgsConstructor
public abstract class AbstractBidirectionalRelation implements BidirectionalRelation {
protected final TextEntity a;
protected final TextEntity b;
@Override
public abstract BidirectionalRelation invert();
}

7
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java
View File

@ -1,7 +0,0 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
public interface BidirectionalRelation extends Relation {
BidirectionalRelation invert();
}

10
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java
View File

@ -1,10 +1,18 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
public class Containment extends AbstractRelation {
public class Containment extends Intersection {
public Containment(TextEntity container, TextEntity contained) {
super(container, contained);
}
public TextEntity getContainer() {
return a;
}
public TextEntity getContained() {
return b;
}
}

3
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java
View File

@ -6,5 +6,6 @@ public enum EntityType {
RECOMMENDATION,
FALSE_POSITIVE,
FALSE_RECOMMENDATION,
DICTIONARY_REMOVAL
DICTIONARY_REMOVAL,
TEMPORARY
}

9
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java
View File

@ -1,17 +1,10 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
public class Equality extends AbstractBidirectionalRelation {
public class Equality extends Containment {
public Equality(TextEntity a, TextEntity b) {
super(a, b);
}
@Override
public Equality invert() {
return new Equality(b, a);
}
}

16
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java
View File

@ -51,16 +51,18 @@ public interface IEntity {
String type();
/**
* Marks this entity and all its intersecting nodes as updated
*/
void update();
/**
* Marks this entity and all its intersecting nodes as removed
*/
void remove();
/**
* An Entity is valid, when it active and not a false recommendation, a false positive or a dictionary removal.
*
@ -349,12 +351,18 @@ public interface IEntity {
*/
default void addMatchedRule(MatchedRule matchedRule) {
boolean valid = valid();
boolean wasValid = valid();
getMatchedRuleList().add(matchedRule);
if (valid() == valid) {
handleStateChange(wasValid);
}
default void handleStateChange(boolean wasValid) {
if (valid() == wasValid) {
return;
}
if(!removed()) {
if (!removed()) {
update();
} else {
remove();

9
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java
View File

@ -1,17 +1,10 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
public class Intersection extends AbstractBidirectionalRelation {
public class Intersection extends AbstractRelation {
public Intersection(TextEntity a, TextEntity b) {
super(a, b);
}
@Override
public Intersection invert() {
return new Intersection(b, a);
}
}

1
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java
View File

@ -6,4 +6,5 @@ public interface Relation {
TextEntity getB();
}

79
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java
View File

@ -1,79 +0,0 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
@Service
public class RelationStore {
//
// private final Map<Pair<String, String>, Relation> relations = new HashMap<>();
private final Map<String, Set<Relation>> entityRelations = new HashMap<>();
public void addRelation(Relation relation) {
String idA = relation.getA().getId();
String idB = relation.getB().getId();
Pair<String, String> key = Pair.of(idA, idB);
// relations.put(key, relation);
if (relation instanceof BidirectionalRelation bidirectionalRelation) {
// BidirectionalRelation invertedRelation = bidirectionalRelation.invert();
// Pair<String, String> invertedKey = Pair.of(idB, idA);
// relations.put(invertedKey, invertedRelation);
entityRelations.computeIfAbsent(idB, k -> new HashSet<>()).add(relation);
}
entityRelations.computeIfAbsent(idA, k -> new HashSet<>()).add(relation);
}
public void removeRelation(Relation relation) {
String idA = relation.getA().getId();
String idB = relation.getB().getId();
Pair<String, String> key = Pair.of(idA, idB);
// relations.remove(key);
if (relation instanceof BidirectionalRelation) {
// Pair<String, String> invertedKey = Pair.of(idB, idA);
// relations.remove(invertedKey);
entityRelations.getOrDefault(idB, Collections.emptySet()).remove(relation);
}
entityRelations.getOrDefault(idA, Collections.emptySet()).remove(relation);
}
public Set<Relation> getRelations(String entityId) {
return entityRelations.getOrDefault(entityId, Collections.emptySet());
}
//
// public Relation getRelation(String idA, String idB) {
//
// return relations.get(Pair.of(idA, idB));
// }
public Set<Relation> getAllRelations(Set<String> entityIds) {
return entityRelations.entrySet()
.stream()
.filter(entry -> entityIds.contains(entry.getKey()))
.flatMap(entry -> entry.getValue()
.stream())
.collect(Collectors.toSet());
}
}

28
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -12,7 +12,10 @@ import java.util.Optional;
import java.util.PriorityQueue;
import java.util.Set;
import org.apache.commons.collections4.map.HashedMap;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
@ -72,8 +75,8 @@ public class TextEntity implements IEntity {
List<SemanticNode> intersectingNodes = new LinkedList<>();
SemanticNode deepestFullyContainingNode;
// @Builder.Default
// Set<Relation> relations = new HashSet<>();
@Builder.Default
HashedMap<TextEntity, Relation> relations = new HashedMap<>();
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
@ -165,13 +168,15 @@ public class TextEntity implements IEntity {
public void removeFromGraph() {
remove("FINAL.0.0", "removed completely");
intersectingNodes.forEach(node -> node.getEntities().remove(this));
pages.forEach(page -> page.getEntities().remove(this));
intersectingNodes = new LinkedList<>();
//relations = new HashSet<>();
relations.keySet()
.forEach(entity -> entity.relations.remove(this));
relations = new HashedMap<>();
deepestFullyContainingNode = null;
pages = new HashSet<>();
remove("FINAL.0.0", "removed completely");
}
@ -259,6 +264,20 @@ public class TextEntity implements IEntity {
}
public void addManualChange(BaseAnnotation manualChange) {
manualOverwrite.addChange(manualChange);
update();
}
public void addManualChanges(List<BaseAnnotation> manualChanges) {
manualOverwrite.addChanges(manualChanges);
update();
}
public boolean matchesAnnotationId(String manualRedactionId) {
return getPositionsOnPagePerPage().stream()
@ -325,6 +344,7 @@ public class TextEntity implements IEntity {
getKieSessionUpdater().ifPresent(updater -> updater.update(this));
}
public void remove() {
getKieSessionUpdater().ifPresent(updater -> updater.remove(this));

95
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java Normal file
View File

@ -0,0 +1,95 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.IntersectingNodeVisitor;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import lombok.experimental.UtilityClass;
@UtilityClass
public class EntityCreationUtility {
public void checkIfBothStartAndEndAreEmpty(String start, String end) {
checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
}
public <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
throw new IllegalArgumentException("Start and end values are empty!");
}
}
public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
if (textBlock.getNextLinebreak(end) < expandedEnd) {
return end;
}
return expandedEnd;
}
public Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
if (initialIntersectingNode.getTextRange().intersects(textRange)) {
initialIntersectingNode.accept(visitor);
}
return visitor.getIntersectingNodes();
}
public void addToPages(TextEntity entity) {
Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
entity.getPages().addAll(pages);
pages.forEach(page -> page.getEntities().add(entity));
}
public void addEntityToNodeEntitySets(TextEntity entity) {
entity.getIntersectingNodes()
.forEach(node -> node.getEntities().add(entity));
}
public boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) {
return true;
}
TextEntity previousEntity = entitiesToMerge.get(0);
for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
boolean typeMatches = textEntity.type().equals(previousEntity.type());
boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
boolean intersects = textEntity.intersects(previousEntity);
if (!typeMatches || !entityTypeMatches || !intersects) {
return false;
}
}
return true;
}
public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
if (textBlock.getTextRange().end() == textRange.end()) {
return new TextRange(textRange.end(), textRange.end());
}
return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
}
}

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java → redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java
View File

@ -1,39 +1,36 @@
package com.iqser.red.service.redaction.v1.server.service.document;
package com.iqser.red.service.redaction.v1.server.utils;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import lombok.RequiredArgsConstructor;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.experimental.UtilityClass;
@Service
@RequiredArgsConstructor
@UtilityClass
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class EntityEnrichmentService {
private final RedactionServiceSettings redactionServiceSettings;
int SURROUNDING_WORDS_OFFSET_WINDOW = 100;
int NUMBER_OF_SURROUNDING_WORDS = 3;
public void enrichEntity(TextEntity entity, TextBlock textBlock) {
entity.setValue(textBlock.subSequence(entity.getTextRange()).toString());
entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock));
entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock));
}
private String findTextAfter(int index, TextBlock textBlock) {
int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end());
int endOffset = Math.min(index + SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().end());
String textAfter = textBlock.subSequence(index, endOffset).toString();
if (!textAfter.isBlank()) {
List<String> wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter);
int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords());
int numberOfWordsAfter = Math.min(wordsAfter.size(), NUMBER_OF_SURROUNDING_WORDS);
if (!wordsAfter.isEmpty()) {
return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" "));
}
@ -41,14 +38,12 @@ public class EntityEnrichmentService {
return "";
}
private String findTextBefore(int index, TextBlock textBlock) {
int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start());
int offsetBefore = Math.max(index - SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().start());
String textBefore = textBlock.subSequence(offsetBefore, index).toString();
if (!textBefore.isBlank()) {
List<String> wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore);
int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords());
int numberOfWordsBefore = Math.min(wordsBefore.size(), NUMBER_OF_SURROUNDING_WORDS);
if (!wordsBefore.isEmpty()) {
return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" "));
}
@ -56,36 +51,26 @@ public class EntityEnrichmentService {
return "";
}
private static List<String> splitToWordsAndRemoveEmptyWords(String textAfter) {
return Arrays.stream(textAfter.split(" "))
private List<String> splitToWordsAndRemoveEmptyWords(String text) {
return Arrays.stream(text.split(" "))
.filter(word -> !Objects.equals("", word))
.toList();
}
private static String concatWordsBefore(List<String> words, boolean endWithSpace) {
private String concatWordsBefore(List<String> words, boolean endWithSpace) {
StringBuilder sb = new StringBuilder();
for (String word : words) {
sb.append(word).append(" ");
}
String result = sb.toString().trim();
return endWithSpace ? result + " " : result;
}
private static String concatWordsAfter(List<String> words, boolean startWithSpace) {
private String concatWordsAfter(List<String> words, boolean startWithSpace) {
StringBuilder sb = new StringBuilder();
for (String word : words) {
sb.append(word).append(" ");
}
String result = sb.toString().trim();
return startWithSpace ? " " + result : result;
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java
View File

@ -11,6 +11,7 @@ import org.apache.commons.lang3.StringUtils;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
@ -44,6 +45,13 @@ public class Dictionary {
}
public boolean containsType(String type) {
Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
return !(levelMap == null || levelMap.isEmpty());
}
private Level getLevel(boolean isDossierDictionary) {
return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE;

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -8,10 +8,8 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
@ -25,9 +23,6 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DictionarySearchService {
EntityEnrichmentService entityEnrichmentService;
RelationStore relationStore;
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
@ -39,7 +34,7 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
EntityCreationService entityCreationService = new EntityCreationService();
dictionary.getDictionarySearch().getBoundaries(node.getTextBlock())
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange()))
.forEach(match -> {

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java
View File

@ -14,7 +14,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
@ -24,14 +23,12 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import lombok.extern.slf4j.Slf4j;
@ -40,15 +37,7 @@ import lombok.extern.slf4j.Slf4j;
@Service
public class EntityFindingUtility {
EntityCreationService entityCreationService;
@Autowired
public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
}
EntityCreationService entityCreationService = new EntityCreationService();
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
Map<String, List<TextEntity>> entitiesWithSameValue,
@ -192,7 +181,7 @@ public class EntityFindingUtility {
return textBlocks.stream()
.flatMap(searchImplementation::getBoundaries)
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, node, Collections.emptySet()))
.filter(Optional::isPresent)
.map(Optional::get)
.distinct()
@ -219,7 +208,7 @@ public class EntityFindingUtility {
return textBlocks.stream()
.flatMap(tb -> searchImplementation.getBoundaries(tb)
.filter(textRange -> entityCreationService.isValidEntityTextRange(tb, textRange)))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet()))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet()))
.filter(Optional::isPresent)
.map(Optional::get)
.distinct()
@ -233,7 +222,7 @@ public class EntityFindingUtility {
return searchImplementation.getBoundaries(document.getTextBlock())
.filter(textRange -> entityCreationService.isValidEntityTextRange(document.getTextBlock(), textRange))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet()))
.map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet()))
.filter(Optional::isPresent)
.map(Optional::get)
.distinct()

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -431,12 +431,11 @@ public class EntityLogCreatorService {
private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) {
case ENTITY -> EntryType.ENTITY;
case ENTITY, TEMPORARY -> EntryType.ENTITY;
case HINT -> EntryType.HINT;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case FALSE_POSITIVE, DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
};
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -17,6 +17,8 @@ import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@ -44,18 +46,19 @@ public class ManualChangesApplicationService {
public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
entityToBeReCategorized.getMatchedRuleList().clear();
entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
if (manualRecategorization.getType() == null) {
return;
}
if (entityToBeReCategorized instanceof Image image) {
entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
return;
}
if (entityToBeReCategorized instanceof TextEntity textEntity) {
textEntity.addManualChange(manualRecategorization);
textEntity.setType(manualRecategorization.getType());
}
}
@ -87,7 +90,7 @@ public class ManualChangesApplicationService {
.map(ManualChangesApplicationService::toRectangle2D)
.collect(Collectors.toList()));
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
entityToBeResized.addManualChange(manualResizeRedaction);
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);

87
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -1,17 +1,10 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
@ -29,44 +22,26 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen
import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
public class EntityCreationService {
private final EntityEnrichmentService entityEnrichmentService;
private final RelationStore relationStore;
private final Set<SemanticNode> nodesInKieSession; // empty set means all nodes are in kieSession
public EntityCreationService(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) {
this.entityEnrichmentService = entityEnrichmentService;
this.relationStore = relationStore;
this.nodesInKieSession = Collections.emptySet();
}
/**
* Creates entities found between specified start and stop strings, case-sensitive.
@ -1010,12 +985,14 @@ public class EntityCreationService {
}
return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
}
boolean added = addEntityToGraph(entity, node.getDocumentTree());
boolean added = node.getDocumentTree().addEntityToGraph(entity);
if (!added) {
return Optional.empty();
}
entity.addEngines(engines);
insertToKieSession(entity, node);
if(!entityType.equals(EntityType.TEMPORARY)) {
insertToKieSession(entity, node);
}
return Optional.of(entity);
}
@ -1080,14 +1057,14 @@ public class EntityCreationService {
.map(TextEntity::getManualOverwrite)
.map(ManualChangeOverwrite::getManualChangeLog)
.flatMap(Collection::stream)
.forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange));
.forEach(mergedEntity::addManualChange);
mergedEntity.setDictionaryEntry(entitiesToMerge.stream()
.anyMatch(TextEntity::isDictionaryEntry));
mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream()
.anyMatch(TextEntity::isDossierDictionaryEntry));
entityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock());
EntityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock());
addEntityToGraph(mergedEntity, node);
insertToKieSession(mergedEntity, node);
@ -1145,7 +1122,7 @@ public class EntityCreationService {
TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
"No entity present!"));
newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog());
newEntity.addManualChanges(entity.getManualOverwrite().getManualChangeLog());
newEntity.setDictionaryEntry(entity.isDictionaryEntry());
newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry());
return newEntity;
@ -1456,7 +1433,7 @@ public class EntityCreationService {
.ifPresent(e -> addDuplicateEntityToGraph(e, entity.getTextRange(), node));
} else {
addEntityToGraph(entity, documentTree);
documentTree.addEntityToGraph(entity);
}
}
@ -1507,48 +1484,4 @@ public class EntityCreationService {
});
}
private boolean addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity);
// if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
// .stream()
// .anyMatch(node -> !nodesInKieSession.contains(node))) {
// entity.removeFromGraph();
// return false;
// }
TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
entityEnrichmentService.enrichEntity(entity, textBlock);
addToPages(entity);
addEntityToNodeEntitySets(entity);
if(relationStore != null) {
//List<Relation> relations = new LinkedList<>();
// compute relationships, by looping through deepestFullyContainingNode's entities
for (TextEntity textEntity : entity.getDeepestFullyContainingNode().getEntities()) {
if (entity.intersects(textEntity) && !entity.equals(textEntity)) {
Relation relation;
if (textEntity.getTextRange().equals(entity.getTextRange())) {
relation = new Equality(entity, textEntity);
} else if (textEntity.containedBy(entity)) {
relation = new Containment(textEntity, entity);
} else if (entity.containedBy(textEntity)) {
relation = new Containment(entity, textEntity);
} else {
relation = new Intersection(entity, textEntity);
}
relationStore.addRelation(relation);
//relations.add(relation);
//textEntity.getRelations().add(relation);
}
}
//entity.getRelations().addAll(relations);
}
return true;
}
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java
View File

@ -9,15 +9,18 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import lombok.experimental.UtilityClass;
@UtilityClass
public class EntityCreationUtility {
public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
public void checkIfBothStartAndEndAreEmpty(String start, String end) {
checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
}
public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
public <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
throw new IllegalArgumentException("Start and end values are empty!");
@ -25,7 +28,7 @@ public class EntityCreationUtility {
}
public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
if (textBlock.getNextLinebreak(end) < expandedEnd) {
return end;
@ -34,7 +37,7 @@ public class EntityCreationUtility {
}
public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
public Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
@ -46,7 +49,7 @@ public class EntityCreationUtility {
}
public static void addToPages(TextEntity entity) {
public void addToPages(TextEntity entity) {
Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
entity.getPages().addAll(pages);
@ -54,14 +57,14 @@ public class EntityCreationUtility {
}
public static void addEntityToNodeEntitySets(TextEntity entity) {
public void addEntityToNodeEntitySets(TextEntity entity) {
entity.getIntersectingNodes()
.forEach(node -> node.getEntities().add(entity));
}
public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
public boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) {
return true;
@ -79,7 +82,7 @@ public class EntityCreationUtility {
}
public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
if (textBlock.getTextRange().end() == textRange.end()) {
return new TextRange(textRange.end(), textRange.end());

185
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools;
import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -11,7 +10,6 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession;
@ -30,13 +28,10 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
@ -54,8 +49,6 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class EntityDroolsExecutionService {
EntityEnrichmentService entityEnrichmentService;
RelationStore relationStore;
ObservationRegistry observationRegistry;
ManualChangesApplicationService manualChangesApplicationService;
RedactionServiceSettings settings;
@ -99,113 +92,87 @@ public class EntityDroolsExecutionService {
KieSession kieSession = kieContainer.newKieSession();
Set<SemanticNode> nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
.count() ? new HashSet<>() : buildSet(sectionsToAnalyze, document);
KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession, relationStore, nodesInKieSession);
document.setKieSessionUpdater(kieSessionUpdater);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore, nodesInKieSession);
RulesLogger logger = new RulesLogger(webSocketService, context);
if (settings.isDroolsDebug()) {
logger.enableAgendaTracking();
logger.enableObjectTracking();
}
kieSession.addEventListener(new TrackingAgendaEventListener(logger));
kieSession.addEventListener(new ObjectTrackingEventListener(logger));
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("dictionary", dictionary);
if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) {
kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
}
nodesInKieSession.add(document);
nodesInKieSession.addAll(sectionsToAnalyze);
nodesInKieSession.addAll(sectionsToAnalyze.stream()
.flatMap(SemanticNode::streamAllSubNodes)
.toList());
nodesInKieSession.forEach(kieSession::insert);
System.out.println("after document insert : " + kieSession.getFactCount());
document.getEntities()
.forEach(kieSession::insert);
relationStore.getAllRelations(document.getEntities()
.stream()
.map(TextEntity::getId)
.collect(Collectors.toSet()))
.forEach(kieSession::insert);
System.out.println("after getEntities insert : " + kieSession.getFactCount());
document.getPages()
.forEach(kieSession::insert);
System.out.println("after getPages insert : " + kieSession.getFactCount());
fileAttributes.stream()
.filter(f -> f.getValue() != null)
.forEach(kieSession::insert);
System.out.println("after fileAttributes insert : " + kieSession.getFactCount());
if (manualRedactions != null) {
manualRedactions.buildAll()
.stream()
.filter(BaseAnnotation::isLocal)
.forEach(kieSession::insert);
}
System.out.println("after manualRedactions insert : " + kieSession.getFactCount());
kieSession.insert(nerEntities);
System.out.println("after nerEntities insert : " + kieSession.getFactCount());
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
kieSession.fireAllRules();
return null;
});
try {
completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
//
// nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
// .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
//
// kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession);
//
// document.setKieSessionUpdater(kieSessionUpdater);
//
// kieSession.insert(document);
//
// kieSession.setGlobal("entityCreationService", entityCreationService);
//
// sectionsToAnalyze.stream()
// .flatMap(SemanticNode::streamAllSubNodes)
// .forEach(kieSession::insert);
// System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
//
// completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession);
document.setKieSessionUpdater(kieSessionUpdater);
} catch (ExecutionException e) {
logger.error(e, "Exception during rule execution");
kieSession.dispose();
if (e.getCause() instanceof TimeoutException) {
EntityCreationService entityCreationService = new EntityCreationService();
RulesLogger logger = new RulesLogger(webSocketService, context);
if (settings.isDroolsDebug()) {
logger.enableAgendaTracking();
logger.enableObjectTracking();
}
kieSession.addEventListener(new TrackingAgendaEventListener(logger));
kieSession.addEventListener(new ObjectTrackingEventListener(logger));
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("dictionary", dictionary);
if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) {
kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
}
kieSession.insert(document);
sectionsToAnalyze.forEach(kieSession::insert);
sectionsToAnalyze.stream()
.flatMap(SemanticNode::streamAllSubNodes)
.forEach(kieSession::insert);
System.out.println("after document insert : " + kieSession.getFactCount());
document.getEntities()
.forEach(kieSession::insert);
document.getEntities()
.forEach(textEntity -> textEntity.getRelations().values()
.forEach(kieSession::insert));
System.out.println("after getEntities insert : " + kieSession.getFactCount());
document.getPages()
.forEach(kieSession::insert);
System.out.println("after getPages insert : " + kieSession.getFactCount());
fileAttributes.stream()
.filter(f -> f.getValue() != null)
.forEach(kieSession::insert);
System.out.println("after fileAttributes insert : " + kieSession.getFactCount());
if (manualRedactions != null) {
manualRedactions.buildAll()
.stream()
.filter(BaseAnnotation::isLocal)
.forEach(kieSession::insert);
}
System.out.println("after manualRedactions insert : " + kieSession.getFactCount());
kieSession.insert(nerEntities);
System.out.println("after nerEntities insert : " + kieSession.getFactCount());
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
CompletableFuture<Void> completableFuture = CompletableFuture.runAsync(kieSession::fireAllRules);
try {
completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
} catch (ExecutionException e) {
logger.error(e, "Exception during rule execution");
if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
logger.error(e, "Exception during rule execution");
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
logger.error(e, "Exception during rule execution");
kieSession.dispose();
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY);
}
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
kieSession.dispose();
return resultingFileAttributes;
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
return resultingFileAttributes;
} finally {
kieSession.dispose();
}
}

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java
View File

@ -1,13 +1,9 @@
package com.iqser.red.service.redaction.v1.server.service.drools;
import java.util.Set;
import org.kie.api.runtime.KieSession;
import org.kie.api.runtime.rule.FactHandle;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -21,16 +17,17 @@ import lombok.experimental.FieldDefaults;
public class KieSessionUpdater implements IKieSessionUpdater {
KieSession kieSession;
RelationStore relationStore;
Set<SemanticNode> nodesInKieSession;
public void insert(TextEntity textEntity) {
kieSession.insert(textEntity);
updateIntersectingNodes(textEntity);
relationStore.getRelations(textEntity.getId())
textEntity.getRelations().values()
.forEach(kieSession::insert);
textEntity.getRelations().keySet()
.forEach(k -> kieSession.insert(k.getRelations()
.get(textEntity)));
}
@ -38,11 +35,15 @@ public class KieSessionUpdater implements IKieSessionUpdater {
kieSession.update(kieSession.getFactHandle(textEntity), textEntity);
updateIntersectingNodes(textEntity);
relationStore.getRelations(textEntity.getId())
.forEach(r -> kieSession.update(kieSession.getFactHandle(r), r));
textEntity.getRelations().values()
.forEach(this::updateFactIfPresent);
textEntity.getRelations().keySet()
.forEach(k -> updateFactIfPresent(k.getRelations()
.get(textEntity)));
}
public void update(Image image) {
kieSession.update(kieSession.getFactHandle(image), image);
@ -56,15 +57,14 @@ public class KieSessionUpdater implements IKieSessionUpdater {
public void remove(TextEntity textEntity) {
//test replace all deletes with updates
kieSession.delete(kieSession.getFactHandle(textEntity));
updateIntersectingNodes(textEntity);
relationStore.getRelations(textEntity.getId())
.forEach(r -> {
FactHandle factHandle = kieSession.getFactHandle(r);
if (factHandle != null) {
kieSession.delete(factHandle);
}
});
textEntity.getRelations().values()
.forEach(this::deleteFactIfPresent);
textEntity.getRelations().keySet()
.forEach(k -> deleteFactIfPresent(k.getRelations()
.get(textEntity)));
}
@ -82,9 +82,25 @@ public class KieSessionUpdater implements IKieSessionUpdater {
private void updateIntersectingNodes(TextEntity textEntity) {
textEntity.getIntersectingNodes()
.stream()
.filter(nodesInKieSession::contains)
.forEach(o -> kieSession.update(kieSession.getFactHandle(o), o));
.forEach(this::updateFactIfPresent);
}
private void updateFactIfPresent(Object o) {
FactHandle factHandle = kieSession.getFactHandle(o);
if (factHandle != null) {
kieSession.update(factHandle, o);
}
}
private void deleteFactIfPresent(Object o) {
FactHandle factHandle = kieSession.getFactHandle(o);
if (factHandle != null) {
kieSession.delete(factHandle);
}
}
}

13
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -17,8 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@ -1334,7 +1333,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1358,7 +1357,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1383,7 +1382,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1394,7 +1393,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1438,7 +1437,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

6
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl
View File

@ -18,8 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;

12
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java
View File

@ -7,6 +7,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.wildfly.common.Assert.assertTrue;
import java.util.HashSet;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
@ -14,12 +15,10 @@ import org.junit.jupiter.api.Test;
import org.kie.api.runtime.KieSession;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
@ -31,14 +30,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater;
public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
@Mock
@ -49,7 +44,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
public void createEntityCreationService() {
MockitoAnnotations.initMocks(this);
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
}
@ -93,6 +88,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
public void assertSameEntitiesCantBeCreatedTwice() {
Document document = buildGraph("files/new/crafted document.pdf");
document.setKieSessionUpdater(new KieSessionUpdater(kieSession));
String type = "CBI_author";
assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document)
.isPresent());

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -30,7 +30,6 @@ import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
@ -40,7 +39,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -54,10 +52,6 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
@Autowired
private DictionaryService dictionaryService;
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
@Autowired
@ -71,7 +65,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
@BeforeEach
public void stubClients() {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java
View File

@ -5,32 +5,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
public class SearchImplementationTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
@Test
public void testSearchImplementationWithPunctuation() {
Document document = buildGraph("files/Minimal Examples/TestPunctuation");
SearchImplementation searchImplementation = new SearchImplementation(List.of("Kuhn, J. O."), true);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
EntityCreationService entityCreationService = new EntityCreationService();
List<TextEntity> entities = entityCreationService.bySearchImplementation(searchImplementation, "CBI_author", EntityType.ENTITY, document)
.toList();
assertEquals(2, entities.size());
@ -42,7 +32,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest {
Document document = buildGraph("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections");
SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
EntityCreationService entityCreationService = new EntityCreationService();
List<TextEntity> entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document)
.toList();
assertEquals(2, entities.size());

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java
View File

@ -12,17 +12,14 @@ import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility;
import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
@ -33,12 +30,6 @@ import lombok.SneakyThrows;
public class TableTest extends BuildDocumentIntegrationTest {
private static final boolean DRAW_FILE = false;
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
private static final String TYPE_1 = "type1";
@ -55,7 +46,7 @@ public class TableTest extends BuildDocumentIntegrationTest {
@BeforeEach
public void createTable() {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
String fileName = "files/Minimal Examples/BasicTable.pdf";

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java
View File

@ -23,7 +23,6 @@ import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser;
@ -36,8 +35,6 @@ class DroolsValidationServiceTest {
@MockBean
RulesClient rulesClient;
@MockBean
EntityEnrichmentService entityEnrichmentService;
@MockBean
RedactionServiceSettings redactionServiceSettings;

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -23,7 +23,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit.jupiter.SpringExtension;
@ -51,11 +50,9 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -69,11 +66,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl");
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
@ -81,7 +73,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
@BeforeEach
public void createServices() {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
}

16
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java
View File

@ -15,16 +15,22 @@ import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import lombok.SneakyThrows;
public class ManualChangesIntegrationTest extends RulesIntegrationTest {
@ -225,4 +231,14 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
return new Rectangle((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
}
@Override
@SneakyThrows
protected Document buildGraph(String filename) {
Document document = super.buildGraph(filename);
document.setKieSessionUpdater(new KieSessionUpdater(kieSession));
return document;
}
}

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java
View File

@ -17,19 +17,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
@ -37,7 +30,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
@BeforeEach
public void createServices() {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
}

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -30,24 +30,17 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import lombok.SneakyThrows;
public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
@Autowired
private EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@ -108,7 +101,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
public void createFoundManualRedaction2() {
Document document = buildGraph("files/Minimal Examples/TestPunctuation");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
EntityCreationService entityCreationService = new EntityCreationService();
List<TextEntity> tempEntities = entityCreationService.byString("Kuhn, J. O.", "CBI_author", EntityType.ENTITY, document)
.toList();
@ -163,7 +156,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
private DocumentAndEntity createFoundManualRedaction() {
Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
EntityCreationService entityCreationService = new EntityCreationService();
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
.toList();

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -19,7 +19,6 @@ import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
@ -29,11 +28,9 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
@ -43,10 +40,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private RelationStore relationStore;
private EntityCreationService entityCreationService;
@ -61,7 +54,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
@BeforeEach
public void createEntityCreationService() {
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
}

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java
View File

@ -22,11 +22,9 @@ import org.springframework.context.annotation.Import;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@ExtendWith(MockitoExtension.class)
@Import(RulesIntegrationTest.TestConfiguration.class)
@ -34,10 +32,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
protected static final String RULES = "drools/rules.drl";
@Autowired
protected EntityEnrichmentService entityEnrichmentService;
@Autowired
protected RelationStore relationStore;
@Autowired
protected ManualChangesApplicationService manualChangesApplicationService;
protected EntityCreationService entityCreationService;
@ -85,7 +79,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
Dictionary dict = Mockito.mock(Dictionary.class);
kieSession = kieContainer.newKieSession();
entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore);
entityCreationService = new EntityCreationService();
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("dictionary", dict);

102
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt
View File

@ -1660,104 +1660,4 @@ Zyma SA
Zyma SA, Nyon, Switzerland
Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
Test Ignored Hint CBI_ADDRESS
the
be
to
of
and
a
in
that
have
I
it
for
not
on
with
he
as
you
do
at
this
but
his
by
from
they
we
say
her
she
or
an
will
my
one
all
would
there
their
what
so
up
out
if
about
who
get
which
go
me
when
make
can
like
time
no
just
him
know
take
people
into
year
your
good
some
could
them
see
other
than
then
now
look
only
come
its
over
think
also
back
after
use
two
how
our
work
first
well
way
even
new
want
because
any
these
give
day
most
us
Test Ignored Hint CBI_ADDRESS

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -1135,7 +1134,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1159,7 +1158,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1184,7 +1183,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1195,7 +1194,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1239,7 +1238,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -1899,7 +1898,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1923,7 +1922,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1948,7 +1947,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1959,7 +1958,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -2003,7 +2002,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -1271,7 +1270,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1295,7 +1294,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1320,7 +1319,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1331,7 +1330,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1375,7 +1374,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -862,7 +861,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -886,7 +885,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -911,7 +910,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -922,7 +921,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -966,7 +965,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -183,7 +182,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -207,7 +206,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -232,7 +231,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -243,7 +242,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -287,7 +286,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

34
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -18,11 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment
import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality
import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -1158,7 +1157,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1182,7 +1181,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1207,7 +1206,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1218,7 +1217,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1262,7 +1261,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end
@ -1499,21 +1498,6 @@ rule "X.10.0: remove false positives of ai"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$intersection: Intersection(
$a: a,
$b: b,
$a.engines contains Engine.MANUAL,
$a.active(),
$b.dictionaryEntry,
$b.engines not contains Engine.MANUAL
)
then
$b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
end
rule "X.11.1: Remove non-manual entity which intersects with a manual entity"
salience 64
when

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -254,7 +253,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -278,7 +277,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -303,7 +302,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -314,7 +313,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -345,7 +344,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -333,7 +332,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -357,7 +356,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -382,7 +381,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -393,7 +392,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -437,7 +436,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -233,7 +232,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -257,7 +256,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -282,7 +281,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -293,7 +292,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -337,7 +336,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -746,7 +745,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -770,7 +769,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -795,7 +794,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -806,7 +805,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -850,7 +849,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
@ -1917,7 +1916,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1941,7 +1940,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1966,7 +1965,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1977,7 +1976,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -2022,7 +2021,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

13
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
@ -1338,7 +1337,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
$entityToBeRemoved.addManualChange($idRemoval);
retract($idRemoval);
end
@ -1362,7 +1361,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
$entityToForce.addManualChange($force);
retract($force);
end
@ -1387,7 +1386,7 @@ rule "MAN.3.0: Apply entity recategorization"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1398,7 +1397,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
$entityToBeRecategorized.addManualChange($recategorization);
retract($recategorization);
end
@ -1441,7 +1440,7 @@ rule "MAN.4.1: Apply legal basis change"
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
$entityToBeChanged.addManualChange($legalBasisChange);
retract($legalBasisChange)
end

2
redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt
View File

File diff suppressed because one or more lines are too long

2
redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt
View File

File diff suppressed because one or more lines are too long