Merge branch 'hotfix-bp' into 'release/4.244.x'

Hotfix bp See merge request redactmanager/redaction-service!286
Hotfix bp
2024-02-26 17:55:40 +01:00 · 2024-02-26 17:55:40 +01:00 · 2024-02-23 13:22:53 +01:00 · 2024-02-23 13:22:53 +01:00 · 2024-02-22 16:15:33 +01:00 · 2024-02-22 16:15:32 +01:00
19 changed files with 619 additions and 290 deletions
@@ -12,7 +12,7 @@ plugins {
 description = "redaction-service-server-v1"


-val layoutParserVersion = "0.86.0"
+val layoutParserVersion = "0.91.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
@@ -65,6 +65,7 @@ dependencies {
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")

    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
    testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
        exclude(
            group = "com.iqser.red.service",
@@ -9,6 +9,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;

@@ -64,8 +65,7 @@ public class Table implements SemanticNode {
     */
    public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {

-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                .flatMap(this::streamRow)
                .map(TableCell::getEntities)
@@ -82,8 +82,11 @@ public class Table implements SemanticNode {
     */
    public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {

-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
    }


@@ -96,9 +99,13 @@ public class Table implements SemanticNode {
     */
    public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {

-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -111,9 +118,13 @@ public class Table implements SemanticNode {
     */
    public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {

-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -126,16 +137,33 @@ public class Table implements SemanticNode {
     */
    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {

-        List<Integer> rowsWithEntityOfType = getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.type())))
-                .map(TextEntity::getIntersectingNodes)
-                .filter(node -> node instanceof TableCell)
-                .map(node -> (TableCell) node)
-                .map(TableCell::getRow)
-                .toList();
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .anyMatch(types::contains))
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
+    }

-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+
+    /**
+     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     * Ignores Entity with ignored == true or removed == true.
+     *
+     * @param types type strings to check whether a row contains an entity like them
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     */
+    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
+
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> {
+                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                            .collect(Collectors.toSet());
+                    return entityTypes.containsAll(types);
+                })
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -148,18 +176,43 @@ public class Table implements SemanticNode {
     */
    public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {

-        return IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
-                        .flatMap(Collection::stream)
-                        .filter(TextEntity::active)
-                        .noneMatch(entity -> types.contains(entity.type())))
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .noneMatch(types::contains))
                .flatMap(this::streamRow)
                .map(TableCell::getEntities)
                .flatMap(Collection::stream);
    }


+    /**
+     * Streams all Entities in the given row.
+     *
+     * @param rowNumber the row number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
+
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
+    /**
+     * Streams all Entities in the given col.
+     *
+     * @param colNumber the column number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
+
+        return streamCol(colNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
    /**
     * Returns a TableCell at the provided row and column location.
     *
@@ -173,7 +226,8 @@ public class Table implements SemanticNode {
            throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
        }
        int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
    }


@@ -196,7 +250,7 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamTableCellsWhichContainType(String type) {

-        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.type().equals(type)));
+        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
    }


@@ -222,7 +276,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamCol(int col) {

-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
    }


@@ -234,7 +289,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamRow(int row) {

-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
    }


@@ -258,7 +314,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamHeadersForCell(int row, int col) {

-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
    }


@@ -348,7 +405,9 @@ public class Table implements SemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
        }
        return textBlock;
    }
@@ -11,6 +11,7 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -213,11 +215,13 @@ public class EntityLogCreatorService {
                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
                .containingNodeId(image.getTreeId())
                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(image.getParent().toString()))
                .imageHasTransparency(image.isTransparent())
                .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
                .state(buildEntryState(image))
                .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .engines(getEngines(null, image.getManualOverwrite()))
                .build();

    }
@@ -245,13 +249,12 @@ public class EntityLogCreatorService {
                .textBefore("")
                .startOffset(-1)
                .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(precursorEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(precursorEntity.getEngines())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
+                .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
                //imported is no longer used, frontend should check engines
                //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
                .imported(false)
@@ -271,7 +274,8 @@ public class EntityLogCreatorService {
                .legalBasis(entity.legalBasis())
                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                .type(entity.type())
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(entity.getDeepestFullyContainingNode().toString()))
                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -281,7 +285,7 @@ public class EntityLogCreatorService {
                .startOffset(entity.getTextRange().start())
                .endOffset(entity.getTextRange().end())
                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
                //imported is no longer used, frontend should check engines
                //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
                .imported(false)
@@ -293,6 +297,17 @@ public class EntityLogCreatorService {
    }


+    private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
+
+        Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
+
+        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
+            engines.add(Engine.MANUAL);
+        }
+        return engines;
+    }
+
+
    private boolean isHint(EntityType entityType) {

        return entityType.equals(EntityType.HINT);
@@ -0,0 +1,160 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility.ENTITY_LAYER;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.awt.Color;
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility;
+import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.SneakyThrows;
+
+public class TableTest extends BuildDocumentIntegrationTest {
+
+    private static final boolean DRAW_FILE = false;
+
+    @Autowired
+    private EntityEnrichmentService entityEnrichmentService;
+
+    private EntityCreationService entityCreationService;
+
+    private static final String TYPE_1 = "type1";
+    private static final String TYPE_2 = "type2";
+    private static final String TYPE_3 = "type3";
+    private static final String TYPE_4 = "type4";
+
+    private Table table;
+
+    private Set<TextEntity> entities;
+
+
+    @SneakyThrows
+    @BeforeEach
+    public void createTable() {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+
+        String fileName = "files/Minimal Examples/BasicTable.pdf";
+
+        Document document = buildGraph(fileName);
+
+        table = (Table) document.streamAllSubNodesOfType(NodeType.TABLE)
+                .findAny()
+                .orElseThrow();
+
+        entities = List.of(//
+                           entityCreationService.byString("Cell11", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell21", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell31", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell41", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell51", TYPE_1, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell12", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell32", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell42", TYPE_2, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell23", TYPE_3, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell53", TYPE_3, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell14", TYPE_4, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell34", TYPE_4, EntityType.ENTITY, document))
+                .stream()
+                .flatMap(Function.identity())
+                .collect(Collectors.toSet());
+
+        if (DRAW_FILE) {
+            File file = new File("/tmp/" + Path.of(fileName).getFileName().toString());
+            storageService.downloadTo(TenantContext.getTenantId(),
+                                      RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.VIEWER_DOCUMENT),
+                                      file);
+            ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+
+            var visualizationsOnPage = EntityVisualizationUtility.createVisualizationsOnPage(document.getEntities(), Color.MAGENTA);
+
+            viewerDocumentService.addVisualizationsOnPage(file,
+                                                          file,
+                                                          Visualizations.builder()
+                                                                  .layer(ENTITY_LAYER)
+                                                                  .visualizationsOnPages(visualizationsOnPage)
+                                                                  .layerVisibilityDefaultValue(true)
+                                                                  .build());
+        }
+
+    }
+
+
+    @Test
+    public void testStreamEntitiesWhereRowContainsEntitiesOfType() {
+
+        int type_2_count = table.getEntitiesOfType(TYPE_2).size();
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(2,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of())
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(1)
+                             .count());
+
+        assertEquals(2,
+                     table.streamTextEntitiesInRow(4)
+                             .count());
+
+        assertEquals(5,
+                     table.streamTextEntitiesInCol(1)
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(3)
+                             .count());
+    }
+
+}
@@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
@@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                .filter(entry -> entry.getValue().equals("Oxford University Press"))
                .findFirst()
                .get();
+        assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL));

        var asyaLyon = redactionLog.getEntityLogEntry()
                .stream()
@@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {

        assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
        assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
+        assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL));
    }


@@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
        analyzeService.reanalyze(request);

        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
-        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(annotationId))
+                .findFirst()
+                .orElseThrow();
        assertEquals("Expand to Hint", entityLogEntry.getValue());
        assertEquals(1, entityLogEntry.getPositions().size());
-        assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertEquals(ManualRedactionType.RESIZE,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testAddEngineManualToResizeDictionaryEntry() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        var dictionaryEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
+                .findFirst()
+                .get();
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(dictionaryEntry.getId())
+                .requestDate(OffsetDateTime.now())
+                .value("Image")
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
+                .updateDictionary(true)
+                .build();
+        manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
    }

 }
@@ -0,0 +1,61 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.awt.Color;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.cos.COSName;
+
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.knecon.fforesight.service.viewerdoc.ContentStreams;
+import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
+import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class EntityVisualizationUtility {
+
+    public static final ContentStreams.Identifier ENTITY_LAYER = new ContentStreams.Identifier("Entities", COSName.getPDFName("KNECON_ENTITIES"), true);
+
+
+    public Map<Integer, VisualizationsOnPage> createVisualizationsOnPage(Collection<TextEntity> entity, Color color) {
+
+        Map<Integer, VisualizationsOnPage> visualizations = new HashMap<>();
+        Set<Page> pages = entity.stream()
+                .map(TextEntity::getPages)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toSet());
+
+        pages.forEach(page -> visualizations.put(page.getNumber() - 1, buildVisualizationsOnPage(color, page)));
+
+        return visualizations;
+    }
+
+
+    private static VisualizationsOnPage buildVisualizationsOnPage(Color color, Page page) {
+
+        return VisualizationsOnPage.builder().coloredRectangles(getEntityRectangles(color, page)).build();
+    }
+
+
+    private static List<ColoredRectangle> getEntityRectangles(Color color, Page page) {
+
+        return page.getEntities()
+                .stream()
+                .map(TextEntity::getPositionsOnPagePerPage)
+                .flatMap(Collection::stream)
+                .filter(p -> p.getPage().equals(page))
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .map(r -> new ColoredRectangle(r, color, 1))
+                .toList();
+    }
+
+}
@@ -33,7 +33,9 @@ public class LayoutParsingRequestProvider {
                .textBlockFileStorageId(textBlockFileStorageId)
                .positionBlockFileStorageId(positionBlockFileStorageId)
                .pageFileStorageId(pageFileStorageId)
-                .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
+                .simplifiedTextStorageId(simplifiedTextStorageId)
+                .viewerDocumentStorageId(viewerDocumentStorageId)
+                .visualLayoutParsingFileId(Optional.empty())
                .build();
    }

@@ -882,8 +882,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -911,7 +911,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -924,7 +924,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -936,7 +936,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -949,7 +949,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -959,7 +959,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -967,26 +967,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -157,18 +157,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"

 rule "CBI.3.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.3.1",
                        "Vertebrate found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                );
-            });
    end

 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -207,23 +206,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
            });
    end

-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                        "CBI.4.1",
                        "Vertebrate but a no redaction indicator found",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                );
-            });
    end


@@ -250,22 +247,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red

 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                        "CBI.5.1",
                        "no_redaction_indicator but also redaction_indicator found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                );
-            });
    end


@@ -355,18 +350,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"

 rule "CBI.8.1: Redacted because table row contains must_redact entity"
    when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                );
-            });
    end


@@ -448,7 +442,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
    then
-
        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
          .ifPresent(authorEntity -> {
              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -744,7 +737,7 @@ rule "PII.1.2: Redact typoed Emails with indicator"
    when
        $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
    then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, $section)
+        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -1452,8 +1445,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -1481,7 +1474,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1494,7 +1487,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -1506,7 +1499,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1519,7 +1512,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -1529,7 +1522,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -1537,26 +1530,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -1312,8 +1312,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -1325,7 +1325,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1338,7 +1338,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -1350,7 +1350,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1363,7 +1363,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -1373,7 +1373,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -225,7 +225,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -100,18 +100,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"

 rule "CBI.3.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.3.1",
                        "Vertebrate found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                );
-            });
    end

 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -150,23 +149,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
            });
    end

-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                        "CBI.4.1",
                        "Vertebrate but a no redaction indicator found",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                );
-            });
    end


@@ -193,22 +190,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red

 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                        "CBI.5.1",
                        "no_redaction_indicator but also redaction_indicator found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                );
-            });
    end


@@ -230,18 +225,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"

 rule "CBI.8.1: Redacted because table row contains must_redact entity"
    when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                );
-            });
    end


@@ -295,7 +289,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
    then
-
        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
          .ifPresent(authorEntity -> {
              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -1042,8 +1035,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -1071,7 +1064,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1084,7 +1077,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -1096,7 +1089,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1109,7 +1102,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -1119,7 +1112,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -1127,26 +1120,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -248,8 +248,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -277,7 +277,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -290,7 +290,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -302,7 +302,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -315,7 +315,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -325,7 +325,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -333,26 +333,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -374,8 +374,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -387,7 +387,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -400,7 +400,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -412,7 +412,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -425,7 +425,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -435,7 +435,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -443,26 +443,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -274,8 +274,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -303,7 +303,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -316,7 +316,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -328,7 +328,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -341,7 +341,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -351,7 +351,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -359,26 +359,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -157,20 +157,20 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"

 rule "CBI.3.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.3.1",
                        "Vertebrate found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                );
-            });
    end

+
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
    when
        $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -207,23 +207,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
            });
    end

-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                        "CBI.4.1",
                        "Vertebrate but a no redaction indicator found",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                );
-            });
    end


@@ -250,22 +248,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red

 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                        "CBI.5.1",
                        "no_redaction_indicator but also redaction_indicator found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                );
-            });
    end


@@ -355,18 +351,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"

 rule "CBI.8.1: Redacted because table row contains must_redact entity"
    when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                        "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                );
-            });
    end


@@ -448,7 +443,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
    then
-
        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
          .ifPresent(authorEntity -> {
              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -743,7 +737,7 @@ rule "PII.1.2: Redact typoed Emails with indicator"
    when
        $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
    then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, $section)
+        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -1473,8 +1467,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -1502,7 +1496,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1515,7 +1509,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -1527,7 +1521,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1540,19 +1534,17 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
    end

-
-// Rule unit: X.5
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -1560,27 +1552,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI


 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

-
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
    end


@@ -1458,8 +1458,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
@@ -1471,7 +1471,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1484,7 +1484,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
@@ -1496,7 +1496,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1509,7 +1509,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
@@ -1521,7 +1521,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
@@ -26,8 +26,8 @@ public class RuleFileMigrationTest {
    // Put your redaction service drools paths and dossier-templates paths both RM and DM here
    static final List<String> ruleFileDirs = List.of(
            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            "/home/kschuettler/iqser/fforesight/dossier-templates-v2/",
-            "/home/kschuettler/iqser/redaction/dossier-templates-v2/");
+            "/home/kschuettler/iqser/fforesight/dossier-templates-v2",
+            "/home/kschuettler/iqser/redaction/dossier-templates-v2");


    @Test
@@ -36,7 +36,11 @@ public class RuleFileMigrationTest {
    void migrateAllEntityRules() {

        for (String ruleFileDir : ruleFileDirs) {
-            Files.walk(Path.of(ruleFileDir)).filter(this::isEntityRuleFile).map(Path::toFile).peek(System.out::println).forEach(RuleFileMigrator::migrateFile);
+            Files.walk(Path.of(ruleFileDir))
+                    .filter(this::isEntityRuleFile)
+                    .map(Path::toFile)
+                    .peek(System.out::println)
+                    .forEach(RuleFileMigrator::migrateFile);
        }
    }
Author	SHA1	Message	Date
Kilian Schüttler	8f6b242c40	Merge branch 'hotfix-bp' into 'release/4.244.x' Hotfix bp See merge request redactmanager/redaction-service!286	2024-02-26 17:55:40 +01:00
Kilian Schüttler	12a22bbb47	Hotfix bp	2024-02-26 17:55:40 +01:00
Corina Olariu	e620412f09	Merge branch 'RED-8589-4.0.0' into 'release/4.244.x' RED-8589 - Add "MANUAL" engine to all annotations that has entries in... See merge request redactmanager/redaction-service!287	2024-02-23 13:22:53 +01:00
Corina Olariu	ac38a966c5	RED-8589 - Add "MANUAL" engine to all annotations that has entries in...	2024-02-23 13:22:53 +01:00
Maverick Studer	89227ec850	Merge branch 'RED-8607-bp' into 'release/4.244.x' RED-8607: Higher rank hint removed if overlaps lower rank redaction See merge request redactmanager/redaction-service!284	2024-02-22 16:15:33 +01:00
Maverick Studer	80fdff803d	RED-8607: Higher rank hint removed if overlaps lower rank redaction	2024-02-22 16:15:32 +01:00