RED-6369: Rules Refactor

* added id to ImageNode
* added Manual Redaction Handling to Rules
* minor rename

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java

@@ -10,7 +10,7 @@ import lombok.NoArgsConstructor;
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class EntityRecogintionEntity {
+public class EntityRecognitionEntity {
 
     private String value;
     private int startOffset;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java

@@ -14,6 +14,6 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class NerEntities {
 
-    private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
+    private Map<Integer, List<EntityRecognitionEntity>> data = new HashMap<>();
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java

@@ -161,9 +161,9 @@ public class EntityNode {
     }
 
 
-    public boolean matchesResizedAnnotationId(String resizedAnnotationId) {
+    public boolean matchesAnnotationId(String manualRedactionId) {
 
-        return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId));
+        return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java

@@ -4,6 +4,7 @@ import static java.lang.String.format;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 
+import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -37,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -254,10 +256,12 @@ public class DocumentGraphFactory {
 
     private void addImage(SectionNode sectionNode, PdfImage image, Context context) {
 
+        Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
         PageNode page = getPage(image.getPage(), context);
         ImageNode imageNode = ImageNode.builder()
+                .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                 .imageType(image.getImageType())
-                .position(RectangleTransformations.toRectangle2D(image.getPosition()))
+                .position(position)
                 .transparency(image.isHasTransparency())
                 .page(page)
                 .tableOfContents(context.tableOfContents())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java

@@ -112,6 +112,12 @@ public class RectangleTransformations {
     }
 
 
+    public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
+
+        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
+    }
+
+
     private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java

@@ -30,6 +30,7 @@ import lombok.experimental.FieldDefaults;
 public class ImageNode implements SemanticNode {
 
     List<Integer> tocId;
+    String id;
 
     ImageType imageType;
     boolean transparency;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java

@@ -19,6 +19,7 @@ public class PropertiesMapper {
         properties.put("imageType", image.getImageType().toString());
         properties.put("transparency", String.valueOf(image.isTransparency()));
         properties.put("position", RectangleTransformations.toString(image.getPosition()));
+        properties.put("id", image.getId());
         return properties;
     }
 
@@ -54,6 +55,7 @@ public class PropertiesMapper {
         builder.imageType(parseImageType(properties.get("imageType")));
         builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
         builder.position(parseRectangle2D(properties.get("position")));
+        builder.id(properties.get("id"));
     }
 
 
@@ -73,7 +75,7 @@ public class PropertiesMapper {
     }
 
 
-    private static ImageType parseImageType(String imageType) {
+    public static ImageType parseImageType(String imageType) {
 
         return switch (imageType) {
             case "LOGO" -> ImageType.LOGO;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java

@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.NoSuchElementException;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ManualRedactionApplicationService {
+
+    private final EntityCreationService entityCreationService;
+
+
+    public void resizeEntityAndReinsert(EntityNode entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
+
+        RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getValue()))
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
+
+        redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(RectangleTransformations::toRectangle2D).toList());
+
+        int newStartOffset;
+        if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
+            newStartOffset = entityToBeResized.getBoundary().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
+        } else {
+            newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
+        }
+
+        entityToBeResized.setResized(true);
+        entityToBeResized.getBoundary().setStart(newStartOffset);
+        entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
+        entityToBeResized.removeFromGraph();
+        entityCreationService.addEntityToGraph(entityToBeResized, entityToBeResized.getDeepestFullyContainingNode().getTableOfContents());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -3,6 +3,7 @@ package drools
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
 import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
+import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageType;
 
 
 import java.util.List;
@@ -21,9 +22,15 @@ import com.iqser.red.service.redaction.v1.server.document.services.EntityCreatio
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService;
 
 global DocumentGraph document
 global EntityCreationService entityCreationService
+global ManualRedactionApplicationService manualRedactionApplicationService
 global Dictionary dictionary
 
 // --------------------------------------- queries -------------------------------------------------------------------
@@ -32,11 +39,64 @@ query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
+// --------------------------------------- manual redaction rules -------------------------------------------------------------------
+
+rule "Apply manual resize redaction"
+    salience 200
+    when
+        $resizeRedactions: ManualResizeRedaction($id: annotationId)
+        $entityToBeResized: EntityNode(matchesAnnotationId($id))
+    then
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedactions);
+        retract($resizeRedactions);
+        update($entityToBeResized);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Entity"
+    salience 200
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: EntityNode(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.removeFromGraph();
+        retract($entityToBeRemoved);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Image"
+    salience 200
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: ImageNode($id == id)
+    then
+        $entityToBeRemoved.setIgnored(true);
+    end
+
+rule "Apply force redaction"
+    salience 200
+    when
+        ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $entityToForce: EntityNode(matchesAnnotationId($id))
+    then
+        $entityToForce.setLegalBasis($legalBasis);
+        $entityToForce.setRedaction(true);
+        $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+    end
+
+rule "Apply image recategorization"
+    salience 200
+    when
+        ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
+        $image: ImageNode($id == id)
+    then
+        $image.setImageType(parseImageType($imageType));
+    end
+
 // --------------------------------------- merging rules -------------------------------------------------------------------
 
 rule "merge intersecting Entities of same type"
     salience 100
-
     when
         $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
         $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
@@ -51,7 +111,6 @@ rule "merge intersecting Entities of same type"
 
 rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 100
-
     when
         $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
         $entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
@@ -62,7 +121,6 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 100
-
     when
         $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
         $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
@@ -73,7 +131,6 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO
 
 rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 100
-
     when
         $entity: EntityNode($type: type, entityType == EntityType.ENTITY)
         $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
@@ -84,7 +141,6 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 rule "remove Entity of lower rank, when equal boundaries and entityType"
     salience 100
-
     when
         $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
         $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
@@ -98,7 +154,6 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
 rule "run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
-
     when
         DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
     then
@@ -107,12 +162,10 @@ rule "run local dictionary search"
         entityNodes.forEach(entityNode -> insert(entityNode));
     end
 
-
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
 rule "1: Redact CBI Authors (Non vertebrate study)"
     no-loop true
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
@@ -124,7 +177,6 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
 
 rule "2: Redact CBI Authors (Vertebrate study)"
     no-loop true
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
@@ -136,7 +188,6 @@ rule "2: Redact CBI Authors (Vertebrate study)"
 
 rule "3: Don't redact CBI Address (Non vertebrate study)"
     no-loop true
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
@@ -147,7 +198,6 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
 
 rule "4: Redact CBI Address (Vertebrate study)"
     no-loop true
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
@@ -158,7 +208,6 @@ rule "4: Redact CBI Address (Vertebrate study)"
     end
 
 rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
-
     when
         $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
     then
@@ -167,10 +216,8 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
         insert(falsePositive);
     end
 
-
 rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
         authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
     then
@@ -182,7 +229,6 @@ rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and
 
 rule "7: Add CBI_author with \"et al.\" Regex"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
          $section: SectionNode(containsString("et al."))
     then
@@ -193,7 +239,6 @@ rule "7: Add CBI_author with \"et al.\" Regex"
     end
 
 rule "8: Add recommendation for Addresses in Test Organism sections"
-
     when
         //FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
@@ -203,7 +248,6 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
         entities.forEach(entity -> insert(entity));
     end
 
-
 rule "9: Add recommendation for Addresses in Test Animals sections"
 
     when
@@ -216,7 +260,6 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
     end
 
 rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
-
     when
         $section : SectionNode(containsString("BL") || containsString("CT"))
     then
@@ -226,13 +269,11 @@ rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-
 // --------------------------------------- PII rules -------------------------------------------------------------------
 
 
 rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
     no-loop true
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
         $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
@@ -242,11 +283,8 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study
         update($entity)
     end
 
-
-
 rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
     no-loop true
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
@@ -257,7 +295,6 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
     end
 
 rule "12: Redact Emails by RegEx (Non vertebrate study)"
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
         $section: SectionNode(containsString("@"))
@@ -268,7 +305,6 @@ rule "12: Redact Emails by RegEx (Non vertebrate study)"
     end
 
 rule "13: Redact Emails by RegEx (Vertebrate study)"
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode(containsString("@"))
@@ -278,10 +314,8 @@ rule "13: Redact Emails by RegEx (Vertebrate study)"
         entities.forEach(entity -> insert(entity));
     end
 
-
 rule "14: Redact line after contact information (Non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $string: String() from List.of("Contact point:",
@@ -313,10 +347,8 @@ rule "14: Redact line after contact information (Non vertebrate study)"
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-
 rule "15: Redact line after contact information (Vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $string: String() from List.of("Contact point:",
@@ -346,10 +378,8 @@ rule "15: Redact line after contact information (Vertebrate study)"
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-
 rule "16: redact line between contact keywords"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
         not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -364,7 +394,6 @@ rule "16: redact line between contact keywords"
 
 rule "17: redact line between contact keywords"
     agenda-group "LOCAL_DICTIONARY_ADDS"
-
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -378,7 +407,6 @@ rule "17: redact line between contact keywords"
     end
 
 rule "18: Redact Phone and Fax by RegEx"
-
     when
         $section: SectionNode(containsString("Contact") ||
             containsString("Telephone") ||