Merge branch 'RED-8049' into 'master'

RED-8049 - Change TextEntity id to be calculated based on position

Closes RED-8049

See merge request redactmanager/redaction-service!255

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 
 import lombok.Builder;
 import lombok.Getter;
@@ -12,6 +12,6 @@ import lombok.Setter;
 public class ClosestEntity {
 
     private double distance;
-    private TextRange textRange;
+    private TextEntity textEntity;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.model;
 
 import java.util.List;
 import java.util.PriorityQueue;
+import java.util.UUID;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@@ -69,6 +71,7 @@ public class ManualEntity implements IEntity {
                 .build();
     }
 
+
     public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
 
         List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
@@ -90,6 +93,19 @@ public class ManualEntity implements IEntity {
     }
 
 
+    public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
+
+        List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+        return ManualEntity.builder()
+                .id(UUID.randomUUID().toString())
+                .value(manualResizeRedaction.getValue())
+                .entityPosition(rectangleWithPages)
+                .entityType(EntityType.ENTITY)
+                .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
+                .build();
+    }
+
+
     @Override
     public TextRange getTextRange() {
 
@@ -103,6 +119,7 @@ public class ManualEntity implements IEntity {
         return getManualOverwrite().getType().orElse(type);
     }
 
+
     private static EntityType getEntityType(EntryType entryType) {
 
         switch (entryType) {
@@ -124,7 +141,9 @@ public class ManualEntity implements IEntity {
         }
     }
 
+
     private EntityType getEntityType(boolean isHint) {
+
         return isHint ? EntityType.HINT : EntityType.ENTITY;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -307,7 +307,7 @@ public final class MigrationEntity {
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
+                .type(entity.type())
                 .section(redactionLogEntry.getSection())
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -128,19 +128,19 @@ public class Dictionary {
 
     public void recommendEverywhere(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
     }
 
 
     public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
     }
 
 
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -33,13 +34,15 @@ public class TextEntity implements IEntity {
 
     // primary key
     @EqualsAndHashCode.Include
-    final TextRange textRange;
+    final String id;
-    @EqualsAndHashCode.Include
-    final String type;
-    @EqualsAndHashCode.Include
-    final EntityType entityType;
     // primary key end
 
+    TextRange textRange;
+    @Builder.Default
+    List<TextRange> duplicateTextRanges = new ArrayList<>();
+    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
+    final EntityType entityType;
+
     @Builder.Default
     final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
     final ManualChangeOverwrite manualOverwrite;
@@ -62,12 +65,30 @@ public class TextEntity implements IEntity {
     SemanticNode deepestFullyContainingNode;
 
 
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) {
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
-        return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+        return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
     }
 
 
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
+
+        return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+    }
+
+
+    private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
+
+        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
+        return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
+    }
+
+
+    public void addTextRange(TextRange textRange) {
+
+        duplicateTextRanges.add(textRange);
+    }
+
     public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
 
         return intersectingNodes.stream().anyMatch(clazz::isInstance);
@@ -82,13 +103,13 @@ public class TextEntity implements IEntity {
 
     public boolean isType(String type) {
 
-        return this.type.equals(type);
+        return type().equals(type);
     }
 
 
     public boolean isAnyType(List<String> types) {
 
-        return types.contains(type);
+        return types.contains(type());
     }
 
 
@@ -125,7 +146,6 @@ public class TextEntity implements IEntity {
                     .min(Comparator.comparingInt(Page::getNumber))
                     .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
 
-            String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
             positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
         }
         return positionsOnPagePerPage;
@@ -193,7 +213,7 @@ public class TextEntity implements IEntity {
         });
         sb.delete(sb.length() - 2, sb.length());
         sb.append("], type = \"");
-        sb.append(type);
+        sb.append(type());
         sb.append("\", EntityType.");
         sb.append(entityType);
         sb.append("]");
@@ -207,4 +227,11 @@ public class TextEntity implements IEntity {
         return getManualOverwrite().getType().orElse(type);
     }
 
+
+    @Override
+    public String value() {
+
+        return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -19,6 +19,7 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
@@ -27,8 +28,12 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Document implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
+    List<Integer> treeId = Collections.emptyList();
+
     Set<Page> pages;
     DocumentTree documentTree;
     Integer numberOfPages;
@@ -67,13 +72,6 @@ public class Document implements GenericSemanticNode {
     }
 
 
-    @Override
-    public List<Integer> getTreeId() {
-
-        return Collections.emptyList();
-    }
-
-
     @Override
     public void setTreeId(List<Integer> tocId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Footer implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Header implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -24,20 +24,19 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
     SectionIdentifier sectionIdentifier;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -32,8 +32,10 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java

@@ -23,26 +23,23 @@ import lombok.experimental.FieldDefaults;
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Page {
 
+    @EqualsAndHashCode.Include
     Integer number;
     Integer height;
     Integer width;
     Integer rotation;
 
-    @EqualsAndHashCode.Exclude
     List<SemanticNode> mainBody;
-    @EqualsAndHashCode.Exclude
     Header header;
-    @EqualsAndHashCode.Exclude
     Footer footer;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<Image> images = new HashSet<>();
 
 
@@ -57,19 +54,4 @@ public class Page {
 
         return String.valueOf(number);
     }
-
-
-    @Override
-    public int hashCode() {
-
-        return number;
-    }
-
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof Page && o.hashCode() == this.hashCode();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -21,19 +21,18 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -24,19 +24,17 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
-
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
 
     TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
 
@@ -80,8 +81,8 @@ public interface SemanticNode {
      */
     default Set<Page> getPages(TextRange textRange) {
 
-        if (!getTextRange().contains(textRange)) {
+        if (!getTextRange().intersects(textRange)) {
-            throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange()));
+            throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
         }
         return getTextBlock().getPages(textRange);
     }
@@ -247,7 +248,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
     }
 
 
@@ -260,7 +261,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
     }
 
 
@@ -273,7 +274,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).map(TextEntity::getType).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
+        return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
     }
 
 
@@ -286,7 +287,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
     }
 
 
@@ -627,6 +628,27 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * For a given TextRange this function returns a List of rectangle around the text in the range.
+     * These Rectangles are split either by a new line or by a large gap in the current line.
+     * This is mainly used to find the positions of TextEntities
+     *
+     * @param textRange A TextRange to calculate the positions for.
+     * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
+     */
+    default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
+
+        if (isLeaf()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
+        if (containingChildNode.isEmpty()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        return containingChildNode.get().getPositionsPerPage(textRange);
+    }
+
+
     /**
      * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
      * If called on the Document, it will return the cropbox of each page
@@ -693,4 +715,18 @@ public interface SemanticNode {
         return bBoxPerPage;
     }
 
+
+    /**
+     * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
+     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
+     *
+     * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
+     * @see NodeVisitor
+     */
+    default void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+        streamChildren().forEach(childNode -> childNode.accept(visitor));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -28,8 +28,10 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
 
@@ -39,10 +41,8 @@ public class Table implements SemanticNode {
     TextBlock textBlock;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 
@@ -128,7 +128,7 @@ public class Table implements SemanticNode {
 
         List<Integer> rowsWithEntityOfType = getEntities().stream()
                 .filter(TextEntity::active)
-                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
+                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.type())))
                 .map(TextEntity::getIntersectingNodes)
                 .filter(node -> node instanceof TableCell)
                 .map(node -> (TableCell) node)
@@ -153,7 +153,7 @@ public class Table implements SemanticNode {
                 .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
                         .flatMap(Collection::stream)
                         .filter(TextEntity::active)
-                        .noneMatch(entity -> types.contains(entity.getType())))
+                        .noneMatch(entity -> types.contains(entity.type())))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
@@ -196,7 +196,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamTableCellsWhichContainType(String type) {
 
-        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
+        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.type().equals(type)));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -23,8 +23,10 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;
     int col;
@@ -36,11 +38,9 @@ public class TableCell implements GenericSemanticNode {
 
     TextBlock textBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -71,7 +71,16 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
 
-        return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList();
+        List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
+        for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
+            if (atomicTextBlock.getTextRange().start() > textRange.end()) {
+                break; // early stop, following TextBlocks will never intersect
+            }
+            if (atomicTextBlock.getTextRange().intersects(textRange)) {
+                intersectingAtomicTextBlocks.add(atomicTextBlock);
+            }
+        }
+        return intersectingAtomicTextBlocks;
     }
 
 
@@ -122,13 +131,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
-
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
         return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
     }
 
+
     public TextRange getLineTextRange(int lineNumber) {
 
         if (atomicTextBlocks.size() == 1) {
@@ -144,10 +153,10 @@ public class ConcatenatedTextBlock implements TextBlock {
         return new TextRange(textRange.start(), textRange.start());
     }
 
+
     @Override
     public List<Rectangle2D> getPositions(TextRange stringTextRange) {
 
-
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
 
         if (textBlocks.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -142,44 +142,39 @@ public class EntityLogCreatorService {
     private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<EntityLogEntry> entries = new ArrayList<>();
-        Set<String> processedIds = new HashSet<>();
         document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
         document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
+        notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity)));
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
 
-        List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
+        // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
 
-            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
-            if (processedIds.contains(positionOnPage.getId())) {
-                continue;
-            }
-            processedIds.add(positionOnPage.getId());
-
-            EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
-            entityLogEntries.setId(positionOnPage.getId());
 
             List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
                     .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            entityLogEntries.setPositions(rectanglesPerLine);
+            // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
-            redactionLogEntities.add(entityLogEntries);
+            entityLogEntry.setId(positionOnPage.getId());
+            entityLogEntry.setPositions(rectanglesPerLine);
+            entityLogEntries.add(entityLogEntry);
         }
 
-        return redactionLogEntities;
+        return entityLogEntries;
     }
 
 
@@ -190,8 +185,6 @@ public class EntityLogCreatorService {
         return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
-                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .value(image.value())
                 .type(imageType)
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
@@ -210,13 +203,12 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         boolean isHint = isHint(manualEntity.getEntityType());
         return EntityLogEntry.builder()
                 .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
                 .reason(manualEntity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(manualEntity.legalBasis())
                 .value(manualEntity.value())
@@ -246,17 +238,16 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
         boolean isHint = isHint(entity.getEntityType());
         return EntityLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
+                .type(entity.type())
                 .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
@@ -282,16 +273,6 @@ public class EntityLogCreatorService {
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
-
-        if (!isApplied && !isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-
-    }
-
-
     private EntryState buildEntryState(IEntity entity) {
 
         if (entity.applied() && entity.active()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -1,54 +1,70 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualChangesApplicationService {
 
-    EntityCreationService entityCreationService;
+    static double MATCH_THRESHOLD = 10;
 
-    public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
+    EntityFindingUtility entityFindingUtility;
 
-        if (IEntityToBeReCategorized instanceof Image image) {
+
+    @Deprecated
+    public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
+
+        entityToBeReCategorized.getMatchedRuleList().clear();
+        entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
+
+        if (entityToBeReCategorized instanceof Image image) {
             image.setImageType(ImageType.fromString(manualRecategorization.getType()));
             return;
         }
-        // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
+
-        if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
+        if (entityToBeReCategorized instanceof TextEntity textEntity) {
-            TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
+            textEntity.setType(manualRecategorization.getType());
-            recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
-            recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
-            textEntity.remove("FINAL.0.0", "removed by manual recategorization");
         }
     }
 
 
+    public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
+
+        resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction);
+    }
+
+
+    @Deprecated
     public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
 
         PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
@@ -57,96 +73,62 @@ public class ManualChangesApplicationService {
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
+        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
+                .stream()
+                .map(ManualChangesApplicationService::toRectangle2D)
+                .collect(Collectors.toList()));
 
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
         SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
+        ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction);
-
         // Loop through nodes starting from the deepest fully containing node all the way to the document node
         while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
 
-                for (TextRange textRange : textRanges) {
+            Map<String, List<TextEntity>> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity));
-                    SemanticNode finalNode = node;
+            Optional<TextEntity> closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD);
 
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
+            if (closestEntity.isPresent()) {
-                            .stream()
+                copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
+                possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-                            .collect(Collectors.toList());
+                return;
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
             }
 
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
+            possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+
             if (node.hasParent()) {
                 node = node.getParent();
             } else {
                 break;
             }
         }
+    }
 
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
 
-        // need to reinsert the entity, due to the boundary having changed.
+    private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) {
-        if (newStartOffset > -1) {
+
-            removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
+        Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
-        }
+        Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
+
+        Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
+
+        Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
+        Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
+
+        Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
+
+        entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
+        entityToBeResized.setTextRange(closestEntity.getTextRange());
+        entityToBeResized.setTextAfter(closestEntity.getTextAfter());
+        entityToBeResized.setTextBefore(closestEntity.getTextBefore());
+        entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        entityToBeResized.setValue(closestEntity.getValue());
+        entityToBeResized.setPages(newIntersectingPages);
         entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
     }
 
 
-    public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
-                                        TextRange textRange,
-                                        List<TextEntity> tempEntities,
-                                        ClosestEntity closestEntity) {
-
-        double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
-        if (currentDistance < closestEntity.getDistance()) {
-            closestEntity.setDistance(currentDistance);
-            closestEntity.setTextRange(textRange);
-        }
-    }
-
-
-    private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
-
-        Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
-        double difference = 0;
-        difference +=  position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
-        difference +=  position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
-        difference +=  position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
-        difference +=  position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
-        return difference;
-    }
-
-
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-
-        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
-    }
-
-
     public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
 
         if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -100,11 +100,11 @@ public class RedactionLogCreatorService {
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
         boolean isHint = isHint(entity.getEntityType());
         return RedactionLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
+                .color(getColor(entity.type(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
+                .type(entity.type())
                 .redacted(entity.applied())
                 .isHint(isHint)
                 .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -3,10 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -23,11 +21,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
 import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -62,6 +57,8 @@ public class UnprocessedChangesService {
     final EntityEnrichmentService entityEnrichmentService;
     final ManualEntityCreationService manualEntityCreationService;
     final DictionaryService dictionaryService;
+    final ManualChangesApplicationService manualChangesApplicationService;
+
     EntityCreationService entityCreationService;
 
 
@@ -145,8 +142,21 @@ public class UnprocessedChangesService {
                 continue;
             }
 
-            TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
+            TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get());
-            resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
+            Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
+                    .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId()))
+                    .findFirst();
+            if (optionalManualResizeRedaction.isPresent()) {
+                ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
+                manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction);
+
+                // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
+                // from the graph, so it does not get processed and sent back to persistence-service to update its value.
+                if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) {
+                    correctEntity.removeFromGraph();
+                }
+            }
+
         }
 
         // remove all temp entities from the graph
@@ -154,101 +164,27 @@ public class UnprocessedChangesService {
     }
 
 
-    public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
+    private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
 
-        if (optionalManualResizeRedaction.isEmpty()) {
+        TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
-            return;
-        }
 
-        ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
+        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
+        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
 
-        PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
+        correctEntity.setValue(closestEntity.getValue());
-                .stream()
+        correctEntity.setTextAfter(closestEntity.getTextAfter());
-                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
+        correctEntity.setTextBefore(closestEntity.getTextBefore());
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
+        correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
-
+        correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
-        SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
-
-        // Loop through nodes starting from the deepest fully containing node all the way to the document node
-        while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
-
-                for (TextRange textRange : textRanges) {
-                    SemanticNode finalNode = node;
-
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
-                            .stream()
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
-                            .collect(Collectors.toList());
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
-            }
-
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
-            if (node.hasParent()) {
-                node = node.getParent();
-            } else {
-                break;
-            }
-        }
-
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
-
-        // need to reinsert the entity, due to the boundary having changed.
-        removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
-    }
-
-
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.removeFromGraph();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-        // Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
-        if (newStartOffset > -1) {
-            entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
-        }
-    }
-
-
-    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
 
         correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
         correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
         correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
 
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
-
         return correctEntity;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
 import java.util.Collection;
@@ -18,24 +19,21 @@ import org.kie.api.runtime.KieSession;
 
 import com.google.common.base.Functions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.RequiredArgsConstructor;
@@ -203,6 +201,7 @@ public class EntityCreationService {
         return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
     }
 
+
     public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
 
         checkIfBothStartAndEndAreEmpty(starts, stops);
@@ -233,9 +232,10 @@ public class EntityCreationService {
         return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
+
     public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0);
+        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node, 0);
     }
 
 
@@ -282,22 +282,11 @@ public class EntityCreationService {
     }
 
 
-    private void checkIfBothStartAndEndAreEmpty(String start, String end) {
-        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
-    }
-
-
-    private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
-        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
-            throw new IllegalArgumentException("Start and end values are empty!");
-        }
-    }
-
-
     public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
+                .stream()
+                .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
                 .map(bounds -> byTextRange(bounds, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -309,7 +298,9 @@ public class EntityCreationService {
         TextBlock textBlock = node.getTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(strings, false);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -321,7 +312,9 @@ public class EntityCreationService {
         TextBlock textBlock = node.getTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(strings, true);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -332,7 +325,9 @@ public class EntityCreationService {
 
         TextBlock textBlock = node.getTextBlock();
         return RedactionSearchUtility.findTextRangesByString(string, textBlock)
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -343,7 +338,9 @@ public class EntityCreationService {
 
         TextBlock textBlock = node.getTextBlock();
         return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock)
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -352,7 +349,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
 
-        return tableNode.streamTableCells().flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
+        return tableNode.streamTableCells()
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
                         tableCell,
                         type,
                         entityType,
@@ -545,7 +543,7 @@ public class EntityCreationService {
     public Optional<TextEntity> byPrefixExpansionRegex(TextEntity entity, String regexPattern) {
 
         int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
-        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
@@ -553,18 +551,10 @@ public class EntityCreationService {
 
         int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
         expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
-        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
-    private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
-
-        if (textBlock.getNextLinebreak(end) < expandedEnd) {
-            return end;
-        }
-        return expandedEnd;
-    }
-
     /**
      * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
      * If the document already contains an equal redaction entity, then the original Entity is returned.
@@ -581,6 +571,7 @@ public class EntityCreationService {
         return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE));
     }
 
+
     /**
      * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
      * If the document already contains an equal redaction entity, then the original Entity is returned.
@@ -599,9 +590,21 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
         if (node.getEntities().contains(entity)) {
-            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();
+            Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
+            if (optionalTextEntity.isEmpty()) {
+                return optionalTextEntity; // Entity has been recategorized and should not be created at all.
+            }
+            TextEntity existingEntity = optionalTextEntity.get();
+            if (existingEntity.getTextRange().equals(textRange)) {
+                return optionalTextEntity; // exactly the same entity, return directly
+            }
+            if (!existingEntity.resized()) {
+                addDuplicateEntityToGraph(existingEntity, textRange, node);
+                return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
+            }
+            return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
         }
         addEntityToGraph(entity, node);
         entity.addEngines(engines);
@@ -610,10 +613,23 @@ public class EntityCreationService {
     }
 
 
+    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
+    @Deprecated(forRemoval = true)
     public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
+        addEntityToGraph(entity, node);
+        return entity;
+    }
+
+
+    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
+    @Deprecated(forRemoval = true)
+    public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node, String id) {
+
+        TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, id);
         addEntityToGraph(entity, node);
         return entity;
     }
@@ -631,7 +647,7 @@ public class EntityCreationService {
             return entitiesToMerge.get(0);
         }
 
-        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType);
+        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node);
         mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
         entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
         entitiesToMerge.stream()
@@ -662,6 +678,7 @@ public class EntityCreationService {
         return newEntity;
     }
 
+
     public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
 
         TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
@@ -683,7 +700,8 @@ public class EntityCreationService {
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
+                "No entity present!"));
     }
 
 
@@ -693,6 +711,18 @@ public class EntityCreationService {
     }
 
 
+    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
+
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
+    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
+
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
@@ -702,21 +732,6 @@ public class EntityCreationService {
     }
 
 
-    public TextEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
-
-        TextEntity highlightEntity = TextEntity.initialEntityNode(new TextRange(tableCell.getTextRange().start(), tableCell.getTextRange().start()), type, entityType);
-
-        String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
-        highlightEntity.setPositionsOnPagePerPage(tableCell.getBBox()
-                .entrySet()
-                .stream()
-                .map(entry -> new PositionOnPage(positionId, entry.getKey(), List.of(entry.getValue())))
-                .toList());
-        addEntityToGraph(highlightEntity, tableCell);
-        return highlightEntity;
-    }
-
-
     public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) {
 
         return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange);
@@ -727,9 +742,17 @@ public class EntityCreationService {
 
         DocumentTree documentTree = node.getDocumentTree();
         try {
-            addEntityToGraph(entity, documentTree);
+            if (node.getEntities().contains(entity)) {
-            entity.addIntersectingNode(documentTree.getRoot().getNode());
+                // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
-            documentTree.getRoot().getNode().getEntities().add(entity);
+                node.getEntities().stream()//
+                        .filter(e -> e.equals(entity))//
+                        .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
+                        .findAny()//
+                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
+            } else {
+                entity.addIntersectingNode(documentTree.getRoot().getNode());
+                addEntityToGraph(entity, documentTree);
+            }
         } catch (NoSuchElementException e) {
             entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
             entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
@@ -740,6 +763,33 @@ public class EntityCreationService {
     }
 
 
+    private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
+
+        entityToDuplicate.addTextRange(newTextRange);
+
+        SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
+                .stream()
+                .sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
+                .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
+                        intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
+                .findFirst()
+                .orElse(node.getDocumentTree().getRoot().getNode());
+
+        entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode);
+
+        Set<SemanticNode> additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange);
+
+        additionalIntersectingNodes.forEach(additionalIntersectingNode -> {
+            if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) {
+                return;
+            }
+            additionalIntersectingNode.getEntities().add(entityToDuplicate);
+            additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate));
+            entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
+        });
+    }
+
+
     private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
 
         SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
@@ -757,45 +807,4 @@ public class EntityCreationService {
     }
 
 
-    private static void addToPages(TextEntity entity) {
-
-        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
-        entity.getPages().addAll(pages);
-        pages.forEach(page -> page.getEntities().add(entity));
-    }
-
-
-    private static void addEntityToNodeEntitySets(TextEntity entity) {
-
-        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
-    }
-
-
-    private static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
-
-        if (entitiesToMerge.isEmpty()) {
-            return true;
-        }
-        TextEntity previousEntity = entitiesToMerge.get(0);
-        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
-            boolean typeMatches = textEntity.getType().equals(previousEntity.getType());
-            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
-            boolean intersects = textEntity.intersects(previousEntity);
-            if (!typeMatches || !entityTypeMatches || !intersects) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-
-    private static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
-
-        if (textBlock.getTextRange().end() == textRange.end()) {
-            return new TextRange(textRange.end(), textRange.end());
-        }
-
-        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java

@@ -0,0 +1,91 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+public class EntityCreationUtility {
+
+
+    public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
+
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
+    public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+
+        if (textBlock.getNextLinebreak(end) < expandedEnd) {
+            return end;
+        }
+        return expandedEnd;
+    }
+
+
+    public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
+
+        IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
+
+        if (initialIntersectingNode.getTextRange().intersects(textRange)) {
+            initialIntersectingNode.accept(visitor);
+        }
+
+        return visitor.getIntersectingNodes();
+    }
+
+
+    public static void addToPages(TextEntity entity) {
+
+        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
+        entity.getPages().addAll(pages);
+        pages.forEach(page -> page.getEntities().add(entity));
+    }
+
+
+    public static void addEntityToNodeEntitySets(TextEntity entity) {
+
+        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
+    }
+
+
+    public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        TextEntity previousEntity = entitiesToMerge.get(0);
+        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = textEntity.type().equals(previousEntity.type());
+            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = textEntity.intersects(previousEntity);
+            if (!typeMatches || !entityTypeMatches || !intersects) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
+
+        if (textBlock.getTextRange().end() == textRange.end()) {
+            return new TextRange(textRange.end(), textRange.end());
+        }
+
+        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
@@ -17,6 +18,7 @@ import java.util.stream.Collectors;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
@@ -58,28 +60,28 @@ public class EntityFindingUtility {
             return Optional.empty();
         }
 
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
+        Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
                 .filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(manualEntity.getEntityPosition(), entity)));
+                .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build())
+                .min(Comparator.comparingDouble(ClosestEntity::getDistance));
 
         if (optionalClosestEntity.isEmpty()) {
             log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition());
             return Optional.empty();
         }
 
-        TextEntity closestEntity = optionalClosestEntity.get();
+        ClosestEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(manualEntity.getEntityPosition(), closestEntity);
+        if (closestEntity.getDistance() > matchThreshold) {
-        if (distance > matchThreshold) {
             log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
                     manualEntity.getValue(),
                     manualEntity.getEntityPosition().get(0).pageNumber(),
                     manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
-                    distance,
+                    closestEntity.getDistance(),
                     matchThreshold);
             return Optional.empty();
         }
 
-        return Optional.of(closestEntity);
+        return Optional.of(closestEntity.getTextEntity());
     }
 
 
@@ -97,7 +99,7 @@ public class EntityFindingUtility {
     }
 
 
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+    public static double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
 
         if (originalPositions.size() != countRectangles(entity)) {
             return Double.MAX_VALUE;
@@ -115,7 +117,7 @@ public class EntityFindingUtility {
     }
 
 
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+    private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
 
         return entity.getPositionsOnPagePerPage()
                 .stream()
@@ -128,7 +130,7 @@ public class EntityFindingUtility {
     }
 
 
-    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+    public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
 
         // mirrored coordinates safe comparison
         double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
@@ -163,7 +165,10 @@ public class EntityFindingUtility {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
-                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .distinct()
                 .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java

@@ -0,0 +1,31 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+import lombok.Getter;
+
+public class IntersectingNodeVisitor implements NodeVisitor {
+
+    @Getter
+    private Set<SemanticNode> intersectingNodes;
+    private final TextRange textRange;
+
+    public IntersectingNodeVisitor(TextRange textRange) {
+
+        this.textRange = textRange;
+        this.intersectingNodes = new HashSet<>();
+    }
+
+    @Override
+    public void visit(SemanticNode node) {
+
+        if (node.getTextRange().intersects(textRange)) {
+            intersectingNodes.add(node);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service.document;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -46,17 +47,24 @@ public class ManualEntityCreationService {
     }
 
 
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions,
+    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
-                                                                                     SemanticNode node,
-                                                                                     String dossierTemplateId) {
 
         Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
-        List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd().stream()
+        List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd()
-                .filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) &&
+                .stream()
-                        manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate())))
+                .filter(manualRedactionEntry -> !(idRemovals.stream()
+                        .map(BaseAnnotation::getAnnotationId)
+                        .toList()
+                        .contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
+                        .isBefore(idRemovals.stream()
+                                .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
+                                .findFirst()
+                                .get()
+                                .getRequestDate())))
                 .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
                 .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
+                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .peek(manualEntity -> {
                     if (manualEntity.getEntityType().equals(EntityType.HINT)) {
                         manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
                     } else {
@@ -75,12 +83,12 @@ public class ManualEntityCreationService {
 
         List<ManualEntity> notFoundManualEntities = new LinkedList<>();
         for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
+            Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
-            if (optionalRedactionEntity.isEmpty()) {
+            if (optionalClosestEntity.isEmpty()) {
                 notFoundManualEntities.add(manualEntity);
                 continue;
             }
-            createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange());
+            createCorrectEntity(manualEntity, optionalClosestEntity.get());
         }
         tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
         return notFoundManualEntities;
@@ -90,25 +98,29 @@ public class ManualEntityCreationService {
     /**
      * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
      *
-     * @param manualEntity     The entity identifier for the RedactionEntity.
+     * @param manualEntity  The entity identifier for the RedactionEntity.
-     * @param node             The SemanticNode associated with the RedactionEntity.
+     * @param closestEntity The closest Boundary to the RedactionEntity.
-     * @param closestTextRange The closest Boundary to the RedactionEntity.
      */
-    private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+    private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
 
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+        TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
+
+        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
+        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
+
+        correctEntity.setValue(closestEntity.getValue());
+        correctEntity.setTextAfter(closestEntity.getTextAfter());
+        correctEntity.setTextBefore(closestEntity.getTextBefore());
+
+        correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
+        correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
 
         correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
         correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
         correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-
-        // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java

@@ -0,0 +1,8 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+public interface NodeVisitor {
+
+    void visit(SemanticNode node);
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -42,7 +42,7 @@ public class EntityDroolsExecutionService {
 
     EntityEnrichmentService entityEnrichmentService;
     ObservationRegistry observationRegistry;
-
+    ManualChangesApplicationService manualChangesApplicationService;
     RedactionServiceSettings settings;
 
 
@@ -73,7 +73,6 @@ public class EntityDroolsExecutionService {
 
         KieSession kieSession = kieContainer.newKieSession();
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
 
         kieSession.setGlobal("document", document);
         kieSession.setGlobal("entityCreationService", entityCreationService);
@@ -88,7 +87,7 @@ public class EntityDroolsExecutionService {
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert);
+            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
             manualRedactions.getRecategorizations().forEach(kieSession::insert);
             manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
             manualRedactions.getForceRedactions().forEach(kieSession::insert);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -293,7 +293,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         System.out.println("Finished structure analysis");
@@ -1331,6 +1331,36 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
     }
 
+    @Test
+    @SneakyThrows
+    public void testResizeWithUpdateDictionaryTrue() {
+
+        String pdfFile = "files/new/crafted document.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get();
+
+        request.setManualRedactions(ManualRedactions.builder()
+                        .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                                .updateDictionary(true)
+                                .annotationId(david.getId())
+                                .requestDate(OffsetDateTime.now())
+                                .value("David Ksenia")
+                                .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build()))
+                                        .addToAllDossiers(false)
+                                .build()))
+                .build());
+        analyzeService.reanalyze(request);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get();
+        assertEquals(resizedEntity.getState(), EntryState.APPLIED);
+        assertEquals(resizedEntity.getValue(), "David Ksenia");
+    }
+
 
     private IdRemoval getIdRemoval(String id) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java

@@ -5,16 +5,18 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import org.junit.jupiter.api.Test;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 
 public class TextEntityTest {
 
     @Test
     public void testMatchedRule() {
 
-        TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
+        ManualEntity entity = ManualEntity.builder()
+                .type("PII")
+                .entityType(EntityType.ENTITY)
+                .build();
         entity.skip("CBI.1.0", "");
         entity.skip("CBI.2.0", "");
         entity.skip("CBI.3.0", "");
@@ -29,7 +31,10 @@ public class TextEntityTest {
     @Test
     public void testMatchedRuleWithNonsense() {
 
-        TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
+        ManualEntity entity = ManualEntity.builder()
+                .type("PII")
+                .entityType(EntityType.ENTITY)
+                .build();
         assertThrows(IllegalArgumentException.class, () -> {
             entity.skip("", "");
         });

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java

@@ -49,6 +49,21 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
     }
 
+    @Test
+    public void assertEntitiesAreDuplicatedWithTheirTableCell() {
+
+        Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf");
+        List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList();
+        assertEquals(3, entities.size());
+        assertEquals(1, entities.stream().distinct().count());
+        assertEquals(2, entities.get(0).getDuplicateTextRanges().size());
+
+        var node = entities.get(0).getDeepestFullyContainingNode();
+
+        assertTrue(node.getTextRange().contains(entities.get(0).getTextRange()));
+        assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr)));
+    }
+
 
     @Test
     public void assertCollectAllEntitiesWorks() {
@@ -82,7 +97,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
         return textEntity;
     }
@@ -241,7 +256,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
 
         assertEquals("2.6.1 Summary of ", textEntity.getTextBefore());
@@ -299,7 +314,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
         Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
-import static org.mockito.Mockito.doThrow;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.when;
 
 import java.awt.Color;
@@ -34,6 +34,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -41,16 +45,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
-import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -245,8 +244,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         System.out.printf("%d Searches took %s s, average %.2f ms\n", numberOfRuns, ((float) totalSearchTime / 1000), totalSearchTime / numberOfRuns);
         System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns);
         System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size());
-        assert document.getEntities().size() == foundEntities.size();
+        for (TextEntity entity : document.getEntities()) {
-
+            var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get();
+            assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange()));
+        }
+        assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size();
+        assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size();
         drawAllEntities(filename, document);
     }
 
@@ -300,7 +303,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         searchImplementation.getBoundaries(textBlock, textBlock.getTextRange())
                 .stream()
                 .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
-                .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType))
+                .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document))
                 .forEach(foundEntities::add);
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -4,11 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.when;
+import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.OffsetDateTime;
 import java.util.Comparator;
@@ -121,6 +123,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         loadNerForTest();
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
@@ -147,7 +150,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String filePath = "files/new/crafted document.pdf";
         AnalyzeRequest request = uploadFileToStorage(filePath);
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
-        AnalyzeResult result = analyzeService.analyze(request);
+        analyzeService.analyze(request);
 
         String testEntityValue1 = "Desiree";
         String testEntityValue2 = "Melanie";
@@ -157,7 +160,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
-        TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
+        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
         String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
@@ -165,24 +168,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
                 .get()
                 .getId();
-        List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
-                .get(0)
-                .getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3))
-                .toList();
-        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
-                .annotationId(idToResize)
-                .value(expandedEntityKeyword)
-                .positions(resizedPositions)
-                .requestDate(OffsetDateTime.now())
-                                .updateDictionary(false)
-                .build();
-        manualResizeRedaction.setUpdateDictionary(false);
         ManualRedactions manualRedactions = new ManualRedactions();
-        manualRedactions.getResizeRedactions().add(manualResizeRedaction);
+        manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
+                        .annotationId(idToResize)
+                        .value(expandedEntityKeyword)
+                        .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
+                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
+                        .addToAllDossiers(false)
+                        .updateDictionary(false)
+                        .requestDate(OffsetDateTime.now())
+                .build());
         request.setManualRedactions(manualRedactions);
-        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+        analyzeService.reanalyze(request);
 
         redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
@@ -225,7 +222,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
                 .legalBasis("Something")
-                                .build()));
+                .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
@@ -248,7 +245,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
                 .legalBasis("Manual Legal Basis Change")
-                                .requestDate(OffsetDateTime.now())
+                .requestDate(OffsetDateTime.now())
                 .build())));
 
         analyzeService.reanalyze(request);
@@ -267,6 +264,45 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testManualRedactionIsDuplicatedWithTableCells() throws IOException {
+
+        System.out.println("testManualRedaction");
+        long start = System.currentTimeMillis();
+        String pdfFile = "files/Minimal Examples/Meto1_Page22.pdf";
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        String manualAddId = UUID.randomUUID().toString();
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(manualAddId);
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("name");
+        manualRedactionEntry.setValue("Surface Water");
+        manualRedactionEntry.setReason("Dictionary Request");
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(76.584f).topLeftY(506.56238f).width(57.4329f).height(9.065156f).page(1).build()));
+        manualRedactions.setEntriesToAdd(Set.of(manualRedactionEntry));
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        request.setManualRedactions(manualRedactions);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/" + Path.of(pdfFile).getFileName() + "MANUAL_REDACTION_TEST.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+        var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
+        assertTrue(optionalEntry.isPresent());
+        assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
     @Test
     public void testReCategorizeToVertebrateChangesCbiAuthor() {
 
@@ -296,7 +332,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         ManualRecategorization recategorization = ManualRecategorization.builder()
                 .requestDate(OffsetDateTime.now())
-                                .type("vertebrate")
+                .type("vertebrate")
                 .annotationId(oxfordUniversityPress.getId())
                 .fileId(TEST_FILE_ID)
                 .build();
@@ -345,18 +381,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())
+                .type("manual")
                 .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
                 .positions(List.of(//
                         new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
                         new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
                         new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
-                                .build()));
+                .build()));
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())
                 .value("Expand to Hint")
                 .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
-                                .updateDictionary(false)
+                .updateDictionary(false)
                 .build();
         manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
         request.setManualRedactions(manualRedactions);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -7,12 +7,14 @@ import static org.wildfly.common.Assert.assertFalse;
 import java.awt.geom.Rectangle2D;
 import java.time.OffsetDateTime;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.junit.jupiter.api.Test;
 
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -50,7 +52,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
         assertEquals(biggerEntity.getTextRange(), entity.getTextRange());
         assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
-        assertEquals(biggerEntity.getIntersectingNodes(), entity.getIntersectingNodes());
+        assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
         assertEquals(biggerEntity.getPages(), entity.getPages());
         assertEquals(biggerEntity.getValue(), entity.getValue());
         assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java

@@ -119,7 +119,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         entity.getManualOverwrite().addChange(imageRecategorizationRequest);
         assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
         assertTrue(entity.getManualOverwrite().getRecategorized().get());
-        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.getType()));
+        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -167,14 +167,14 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
 
     private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<TextEntity> entities) {
 
-        return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type)));
+        return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type)));
 
     }
 
 
     private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<TextEntity> entities) {
 
-        return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type))));
+        return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type))));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -28,6 +28,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     protected EntityEnrichmentService entityEnrichmentService;
+    @Autowired
+    protected ManualChangesApplicationService manualChangesApplicationService;
     protected EntityCreationService entityCreationService;
     protected KieSession kieSession;
 
@@ -72,7 +74,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
         kieSession = kieContainer.newKieSession();
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
         kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
         kieSession.setGlobal("entityCreationService", entityCreationService);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -174,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         analyzeService.reanalyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -221,7 +221,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         List<Rectangle> positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build());
         List<Rectangle> positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
@@ -285,7 +285,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -323,7 +323,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -361,7 +361,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -84,7 +84,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -92,7 +92,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -102,7 +102,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -110,7 +110,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -119,7 +119,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -148,7 +148,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
@@ -301,7 +301,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -309,7 +309,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -625,7 +625,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -634,7 +634,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@@ -693,7 +693,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
         $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@@ -709,7 +709,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -733,7 +733,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -811,13 +811,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -825,7 +824,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -883,8 +882,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -895,8 +894,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -911,8 +910,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -924,8 +923,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -936,8 +935,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -959,8 +958,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -971,8 +970,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -982,8 +981,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
@@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
 rule "CBI.13.0: Ignore CBI Address recommendations"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
+        $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION)
     then
         $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
         retract($entity)
@@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+        $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
@@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 rule "CBI.18.0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
-        $entityToExpand: TextEntity(type == "CBI_author",
+        $entityToExpand: TextEntity(type() == "CBI_author",
           value.matches("[^\\s]+"),
           textAfter.startsWith(" "),
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
 // Rule unit: CBI.19
 rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
-        $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
          .ifPresent(expandedEntity -> {
@@ -668,7 +668,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -680,7 +680,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -707,7 +707,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -715,7 +715,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -1047,7 +1047,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1134,21 +1134,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 // Rule unit: ETC.4
 rule "ETC.4.0: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.1: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.2: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No  178/2002)");
     end
@@ -1158,7 +1158,7 @@ rule "ETC.4.2: Redact dossier dictionary entries"
 rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
         $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@@ -1212,7 +1212,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
 rule "ETC.9.0: Redact skipped impurities"
     when
         FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "skipped_impurities")
+        $skippedImpurities: TextEntity(type() == "skipped_impurities")
     then
         $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1220,7 +1220,7 @@ rule "ETC.9.0: Redact skipped impurities"
 rule "ETC.9.1: Redact impurities"
     when
         FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "impurities")
+        $skippedImpurities: TextEntity(type() == "impurities")
     then
         $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1229,7 +1229,7 @@ rule "ETC.9.1: Redact impurities"
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
     when
-        $compositionInformation: TextEntity(type == "product_composition")
+        $compositionInformation: TextEntity(type() == "product_composition")
     then
         $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
     end
@@ -1256,7 +1256,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1278,7 +1278,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author"
     then
         nerEntities.getNerEntityList().stream()
             .filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1289,7 +1289,7 @@ rule "AI.3.0: Recommend authors from AI as PII"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1303,7 +1303,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1381,13 +1381,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -1395,7 +1394,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1453,8 +1452,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1465,8 +1464,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1481,8 +1480,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1494,8 +1493,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1506,8 +1505,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1529,8 +1528,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1541,8 +1540,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -1552,8 +1551,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -466,7 +466,9 @@ rule "DOC.7.1: Performing Laboratory (Country)"
      then
          nerEntities.streamEntitiesOfType("COUNTRY")
                      .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
-                     .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .flatMap(Optional::stream)
+                     .collect(Collectors.toList())
                      .forEach(entity -> {
                           entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
                      });
@@ -475,7 +477,7 @@ rule "DOC.7.1: Performing Laboratory (Country)"
 rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
     when
         $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
-        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
+        $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
     then
         $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
     end
@@ -1161,7 +1163,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1239,13 +1241,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -1253,7 +1254,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1311,8 +1312,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1323,8 +1324,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1336,8 +1337,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1348,8 +1349,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1371,8 +1372,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -75,7 +75,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -153,13 +153,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -167,7 +166,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -225,8 +224,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -339,7 +339,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+        $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
@@ -442,7 +442,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 rule "CBI.18.0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
-        $entityToExpand: TextEntity(type == "CBI_author",
+        $entityToExpand: TextEntity(type() == "CBI_author",
           value.matches("[^\\s]+"),
           textAfter.startsWith(" "),
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@@ -460,7 +460,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
 // Rule unit: CBI.19
 rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
-        $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
          .ifPresent(expandedEntity -> {
@@ -505,7 +505,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -513,7 +513,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -733,7 +733,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -742,7 +742,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@@ -800,7 +800,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 // Rule unit: ETC.4
 rule "ETC.4.0: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -810,7 +810,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
 rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
         $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@@ -869,7 +869,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -893,7 +893,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -971,13 +971,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -985,7 +984,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1043,8 +1042,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1055,8 +1054,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1071,8 +1070,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1084,8 +1083,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1096,8 +1095,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1119,8 +1118,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1131,8 +1130,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -1142,8 +1141,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -71,7 +71,7 @@ query "getFileAttributes"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -83,7 +83,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -98,7 +98,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -112,7 +112,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -190,13 +190,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -204,7 +203,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -249,8 +248,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -261,8 +260,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -277,8 +276,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -290,8 +289,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -302,8 +301,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -325,8 +324,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -337,8 +336,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -348,8 +347,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -225,7 +225,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -303,13 +303,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -317,7 +316,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -375,8 +374,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -387,8 +386,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -400,8 +399,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -412,8 +411,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -435,8 +434,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -447,8 +446,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -458,8 +457,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -125,7 +125,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -203,13 +203,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -217,7 +216,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -275,8 +274,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -287,8 +286,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -303,8 +302,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -316,8 +315,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -328,8 +327,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -351,8 +350,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -363,8 +362,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -374,8 +373,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -81,7 +81,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -89,7 +89,7 @@ rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -99,7 +99,7 @@ rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
 rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -107,7 +107,7 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -116,7 +116,7 @@ rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Don't redact genitive CBI_author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -145,7 +145,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
+        $author: TextEntity(type() == "CBI_author", active()) from $tableCell.getEntities()
     then
         $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
@@ -298,7 +298,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -306,7 +306,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -452,7 +452,7 @@ rule "ETC.3.1: Redact logos (non vertebrate study)"
 rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
         update($dossierRedaction);
@@ -469,7 +469,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -494,7 +494,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         $resizeRedaction: ManualResizeRedaction($id: annotationId)
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -575,8 +575,7 @@ rule "MAN.3.0: Apply entity recategorization"
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
         manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        update($entityToBeRecategorized);
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply image recategorization"
@@ -617,8 +616,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -629,8 +628,8 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 rule "X.1.0: merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !resized(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -645,8 +644,8 @@ rule "X.1.0: merge intersecting Entities of same type"
 rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, entityType == EntityType.ENTITY, !resized(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -658,8 +657,8 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -670,8 +669,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $recommendation: TextEntity(intersects($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@@ -695,8 +694,8 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
+        $higherRank: TextEntity($type: type(), entityType == EntityType.ENTITY, active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
@@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
 rule "CBI.13.0: Ignore CBI Address recommendations"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
+        $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION)
     then
         $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
         retract($entity)
@@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+        $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
@@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 rule "CBI.18.0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
-        $entityToExpand: TextEntity(type == "CBI_author",
+        $entityToExpand: TextEntity(type() == "CBI_author",
           value.matches("[^\\s]+"),
           textAfter.startsWith(" "),
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
 // Rule unit: CBI.19
 rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
-        $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
          .ifPresent(expandedEntity -> {
@@ -667,7 +667,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -679,7 +679,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -706,7 +706,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -714,7 +714,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -1045,7 +1045,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1056,7 +1056,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@@ -1142,21 +1142,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 // Rule unit: ETC.4
 rule "ETC.4.0: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.1: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.2: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No  178/2002)");
     end
@@ -1166,7 +1166,7 @@ rule "ETC.4.2: Redact dossier dictionary entries"
 rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
         $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
@@ -1219,7 +1219,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
 rule "ETC.9.0: Redact skipped impurities"
     when
         FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "skipped_impurities")
+        $skippedImpurities: TextEntity(type() == "skipped_impurities")
     then
         $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1227,7 +1227,7 @@ rule "ETC.9.0: Redact skipped impurities"
 rule "ETC.9.1: Redact impurities"
     when
         FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "impurities")
+        $skippedImpurities: TextEntity(type() == "impurities")
     then
         $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1235,7 +1235,7 @@ rule "ETC.9.1: Redact impurities"
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
     when
-        $compositionInformation: TextEntity(type == "product_composition")
+        $compositionInformation: TextEntity(type() == "product_composition")
     then
         $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
     end
@@ -1256,7 +1256,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne
 rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -1264,7 +1264,7 @@ rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)"
 rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -1278,7 +1278,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1300,7 +1300,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author"
     then
         nerEntities.getNerEntityList().stream()
             .filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1311,7 +1311,7 @@ rule "AI.3.0: Recommend authors from AI as PII"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
     end
 
 //------------------------------------ Manual redaction rules ------------------------------------
@@ -1324,7 +1324,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1402,13 +1402,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -1416,7 +1415,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1474,8 +1473,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1486,8 +1485,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1502,8 +1501,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1515,8 +1514,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1527,8 +1526,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1552,8 +1551,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1564,8 +1563,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
         $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
@@ -1576,8 +1575,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
+        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -464,7 +464,9 @@ rule "DOC.7.1: Performing Laboratory (Country)"
      then
          nerEntities.streamEntitiesOfType("COUNTRY")
                      .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
-                     .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .flatMap(Optional::stream)
+                     .collect(Collectors.toList())
                      .forEach(entity -> {
                           entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
                      });
@@ -473,7 +475,7 @@ rule "DOC.7.1: Performing Laboratory (Country)"
 rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
     when
         $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
-        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
+        $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
     then
         $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
     end
@@ -1308,7 +1310,7 @@ rule "MAN.0.0: Apply manual resize redaction"
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1386,13 +1388,12 @@ rule "MAN.3.0: Apply entity recategorization"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
@@ -1400,7 +1401,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
     when
         $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1457,8 +1458,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1469,8 +1470,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1482,8 +1483,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1494,8 +1495,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1519,8 +1520,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active())
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);