RED-6369: Rules Refactor

* added id to ImageNode
* added Manual Redaction Handling to Rules
* minor rename
This commit is contained in:
Kilian Schuettler 2023-04-19 13:52:27 +02:00
parent 3e7081a87a
commit 8bbaa4c140
9 changed files with 126 additions and 40 deletions

View File

@ -10,7 +10,7 @@ import lombok.NoArgsConstructor;
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecogintionEntity {
public class EntityRecognitionEntity {
private String value;
private int startOffset;

View File

@ -14,6 +14,6 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class NerEntities {
private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
private Map<Integer, List<EntityRecognitionEntity>> data = new HashMap<>();
}

View File

@ -161,9 +161,9 @@ public class EntityNode {
}
public boolean matchesResizedAnnotationId(String resizedAnnotationId) {
public boolean matchesAnnotationId(String manualRedactionId) {
return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId));
return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
}

View File

@ -4,6 +4,7 @@ import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -37,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@ -254,10 +256,12 @@ public class DocumentGraphFactory {
private void addImage(SectionNode sectionNode, PdfImage image, Context context) {
Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
PageNode page = getPage(image.getPage(), context);
ImageNode imageNode = ImageNode.builder()
.id(IdBuilder.buildId(Set.of(page), List.of(position)))
.imageType(image.getImageType())
.position(RectangleTransformations.toRectangle2D(image.getPosition()))
.position(position)
.transparency(image.isHasTransparency())
.page(page)
.tableOfContents(context.tableOfContents())

View File

@ -112,6 +112,12 @@ public class RectangleTransformations {
}
public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
}
private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
@Override

View File

@ -30,6 +30,7 @@ import lombok.experimental.FieldDefaults;
public class ImageNode implements SemanticNode {
List<Integer> tocId;
String id;
ImageType imageType;
boolean transparency;

View File

@ -19,6 +19,7 @@ public class PropertiesMapper {
properties.put("imageType", image.getImageType().toString());
properties.put("transparency", String.valueOf(image.isTransparency()));
properties.put("position", RectangleTransformations.toString(image.getPosition()));
properties.put("id", image.getId());
return properties;
}
@ -54,6 +55,7 @@ public class PropertiesMapper {
builder.imageType(parseImageType(properties.get("imageType")));
builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
builder.position(parseRectangle2D(properties.get("position")));
builder.id(properties.get("id"));
}
@ -73,7 +75,7 @@ public class PropertiesMapper {
}
private static ImageType parseImageType(String imageType) {
public static ImageType parseImageType(String imageType) {
return switch (imageType) {
case "LOGO" -> ImageType.LOGO;

View File

@ -0,0 +1,45 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import java.util.NoSuchElementException;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class ManualRedactionApplicationService {
private final EntityCreationService entityCreationService;
public void resizeEntityAndReinsert(EntityNode entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
.stream()
.filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getValue()))
.findFirst()
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(RectangleTransformations::toRectangle2D).toList());
int newStartOffset;
if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
newStartOffset = entityToBeResized.getBoundary().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
} else {
newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
}
entityToBeResized.setResized(true);
entityToBeResized.getBoundary().setStart(newStartOffset);
entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
entityToBeResized.removeFromGraph();
entityCreationService.addEntityToGraph(entityToBeResized, entityToBeResized.getDeepestFullyContainingNode().getTableOfContents());
}
}

View File

@ -3,6 +3,7 @@ package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageType;
import java.util.List;
@ -21,9 +22,15 @@ import com.iqser.red.service.redaction.v1.server.document.services.EntityCreatio
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService;
global DocumentGraph document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
@ -32,11 +39,64 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
rule "Apply manual resize redaction"
salience 200
when
$resizeRedactions: ManualResizeRedaction($id: annotationId)
$entityToBeResized: EntityNode(matchesAnnotationId($id))
then
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions);
retract($resizeRedactions);
update($entityToBeResized);
end
rule "Apply id removals that are valid and not in forced redactions to Entity"
salience 200
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: EntityNode(matchesAnnotationId($id))
then
$entityToBeRemoved.removeFromGraph();
retract($entityToBeRemoved);
end
rule "Apply id removals that are valid and not in forced redactions to Image"
salience 200
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: ImageNode($id == id)
then
$entityToBeRemoved.setIgnored(true);
end
rule "Apply force redaction"
salience 200
when
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: EntityNode(matchesAnnotationId($id))
then
$entityToForce.setLegalBasis($legalBasis);
$entityToForce.setRedaction(true);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
end
rule "Apply image recategorization"
salience 200
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: ImageNode($id == id)
then
$image.setImageType(parseImageType($imageType));
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type"
salience 100
when
$first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
@ -51,7 +111,6 @@ rule "merge intersecting Entities of same type"
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 100
when
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
@ -62,7 +121,6 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 100
when
$falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
@ -73,7 +131,6 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 100
when
$entity: EntityNode($type: type, entityType == EntityType.ENTITY)
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
@ -84,7 +141,6 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 100
when
$higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
@ -98,7 +154,6 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
@ -107,12 +162,10 @@ rule "run local dictionary search"
entityNodes.forEach(entityNode -> insert(entityNode));
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
@ -124,7 +177,6 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
rule "2: Redact CBI Authors (Vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
@ -136,7 +188,6 @@ rule "2: Redact CBI Authors (Vertebrate study)"
rule "3: Don't redact CBI Address (Non vertebrate study)"
no-loop true
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
@ -147,7 +198,6 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
rule "4: Redact CBI Address (Vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
@ -158,7 +208,6 @@ rule "4: Redact CBI Address (Vertebrate study)"
end
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "[''ʼˈ´`ʻ']s"), redaction)
then
@ -167,10 +216,8 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
insert(falsePositive);
end
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
then
@ -182,7 +229,6 @@ rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and
rule "7: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
@ -193,7 +239,6 @@ rule "7: Add CBI_author with \"et al.\" Regex"
end
rule "8: Add recommendation for Addresses in Test Organism sections"
when
//FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
@ -203,7 +248,6 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
entities.forEach(entity -> insert(entity));
end
rule "9: Add recommendation for Addresses in Test Animals sections"
when
@ -216,7 +260,6 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
end
rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
when
$section : SectionNode(containsString("BL") || containsString("CT"))
then
@ -226,13 +269,11 @@ rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
no-loop true
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
@ -242,11 +283,8 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study
update($entity)
end
rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
@ -257,7 +295,6 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
end
rule "12: Redact Emails by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@"))
@ -268,7 +305,6 @@ rule "12: Redact Emails by RegEx (Non vertebrate study)"
end
rule "13: Redact Emails by RegEx (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("@"))
@ -278,10 +314,8 @@ rule "13: Redact Emails by RegEx (Vertebrate study)"
entities.forEach(entity -> insert(entity));
end
rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:",
@ -313,10 +347,8 @@ rule "14: Redact line after contact information (Non vertebrate study)"
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "15: Redact line after contact information (Vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:",
@ -346,10 +378,8 @@ rule "15: Redact line after contact information (Vertebrate study)"
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@ -364,7 +394,6 @@ rule "16: redact line between contact keywords"
rule "17: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@ -378,7 +407,6 @@ rule "17: redact line between contact keywords"
end
rule "18: Redact Phone and Fax by RegEx"
when
$section: SectionNode(containsString("Contact") ||
containsString("Telephone") ||