Merge branch 'RED-10687-bp' into 'release/4.423.x'

RED-10687: Filter out overlapping recommendations if they are of the same type

See merge request redactmanager/redaction-service!581

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.587.0"
+val persistenceServiceVersion = "2.589.18"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.181.0"
+val layoutParserVersion = "0.191.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.592.0-RED10260.0"
+val persistenceServiceVersion = "2.589.18"
 val llmServiceVersion = "1.11.0"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -125,6 +125,7 @@ public class PrecursorEntity implements IEntity {
                 .id(importedRedaction.getId())
                 .value(value)
                 .entityPosition(rectangleWithPages)
+                .ruleIdentifier("IMP.0.0")
                 .reason(Optional.ofNullable(importedRedaction.getReason())
                                 .orElse(""))
                 .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -30,7 +30,7 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
     public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
     public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE);
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, MANUAL_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java

@@ -9,7 +9,6 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java

@@ -0,0 +1,7 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+public enum LayoutEngine {
+    ALGORITHM,
+    AI,
+    OUTLINE
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -25,7 +25,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 public interface SemanticNode {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -17,7 +17,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -8,6 +8,7 @@ import static java.lang.String.format;
 import java.io.IOException;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.concurrent.CompletionException;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -18,6 +19,7 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@@ -167,13 +169,33 @@ public class RedactionMessageReceiver {
     private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
 
         log.error("Failed to process analyze request: {}", analyzeRequest, e);
-        var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+
+        ErrorCode errorCode = null;
+
+        if(e instanceof CompletionException ce){
+            errorCode = computeErrorCodeFromDroolsTimeoutException(ce.getCause());
+        } else {
+            errorCode = computeErrorCodeFromDroolsTimeoutException(e);
+        }
+
         fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                         analyzeRequest.getFileId(),
                                                         new FileErrorInfo(e.getMessage(),
                                                                           priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE,
                                                                           "redaction-service",
-                                                                          timestamp));
+                                                                          errorCode));
+    }
+
+    private ErrorCode computeErrorCodeFromDroolsTimeoutException(Throwable e){
+        ErrorCode errorCode = null;
+        if(e instanceof DroolsTimeoutException dre){
+            if (!dre.isReported()){
+                errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT;
+            } else {
+                errorCode = ErrorCode.LOCKED_RULES;
+            }
+        }
+        return errorCode;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
@@ -95,11 +96,15 @@ public class DocumentGraphMapper {
             }
             List<Integer> treeId = entryData.getTreeIdList();
             entryData.getEnginesList()
+                    .stream()
+                    .map(Enum::name)
+                    .map(LayoutEngine::valueOf)
                     .forEach(node::addEngine);
             node.setTreeId(treeId);
 
             newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
-        } return newEntries;
+        }
+        return newEntries;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1002,11 +1002,11 @@ public class EntityCreationService {
                 return Optional.empty(); // Entity has been recategorized and should not be created at all.
             }
             TextEntity existingEntity = optionalTextEntity.get();
-            if (existingEntity.getTextRange().equals(textRange)) {
+            if (existingEntity.getTextRange().equals(trimmedTextRange)) {
                 return optionalTextEntity; // exactly the same entity, return directly
             }
             if (!existingEntity.resized()) {
-                addDuplicateEntityToGraph(existingEntity, textRange, node);
+                addDuplicateEntityToGraph(existingEntity, trimmedTextRange, node);
                 return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
             }
             return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -14,12 +14,15 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
@@ -40,9 +43,14 @@ public class EntityFromPrecursorCreationService {
     RedactionServiceSettings settings;
 
 
-    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, Document document, String dossierTemplateId) {
 
+        Set<Integer> pageNumbers = document.getPages()
+                .stream()
+                .map(Page::getNumber)
+                .collect(Collectors.toSet());
         Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
+
         List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
                 .stream()
                 .filter(BaseAnnotation::isLocal)
@@ -51,6 +59,10 @@ public class EntityFromPrecursorCreationService {
                         .filter(idRemoval -> idRemoval.getRequestDate().isAfter(manualRedactionEntry.getRequestDate()))
                         .findAny()//
                         .isEmpty())
+                .filter(manualRedactionEntry -> manualRedactionEntry.getPositions()
+                        .stream()
+                        .map(Rectangle::getPage)
+                        .allMatch(pageNumbers::contains))
                 .map(manualRedactionEntry -> //
                              PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                 .peek(manualEntity -> {
@@ -61,7 +73,7 @@ public class EntityFromPrecursorCreationService {
                     }
                 })
                 .toList();
-        return toTextEntity(manualEntities, node);
+        return toTextEntity(manualEntities, document);
     }
 
 
@@ -130,10 +142,8 @@ public class EntityFromPrecursorCreationService {
         } else {
             String section = precursorEntity.getManualOverwrite().getSection()
                     .orElse(null);
-            if ((section == null || section.isBlank())
-                && precursorEntity.getSection() != null
-                && !precursorEntity.getSection().isBlank()
-                && precursorEntity.getEngines().contains(Engine.IMPORTED)) {
+            if ((section == null || section.isBlank()) && precursorEntity.getSection() != null && !precursorEntity.getSection().isBlank() && precursorEntity.getEngines()
+                    .contains(Engine.IMPORTED)) {
                 section = precursorEntity.getSection();
             }
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl

@@ -55,14 +55,13 @@ global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
 global RulesLogger logger
 
-
 //------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-//------------------------------------ H rules ------------------------------------
+//------------------------------------ Headlines rules ------------------------------------
 
 // Rule unit: H.0
 rule "H.0.0: retract table of contents page"
@@ -129,6 +128,7 @@ rule "H.3.1: Study Type File Attribute in Headlines"
              .ifPresent(fileAttribute -> insert(fileAttribute));
     end
 
+
 //------------------------------------ General documine rules ------------------------------------
 
 // Rule unit: DOC.1
@@ -296,6 +296,7 @@ rule "DOC.1.4: Guideline in Headlines"
         );
     end
 
+
 // Rule unit: DOC.2
 rule "DOC.2.0: Report number"
     when
@@ -1147,6 +1148,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
             .forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
     end
 
+
 //------------------------------------ Table extraction rules ------------------------------------
 
 // Rule unit: TAB.0
@@ -1296,7 +1298,8 @@ rule "TAB.7.0: Indicator (Species)"
             .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1424,6 +1427,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
 
+
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
@@ -1485,7 +1489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -1534,8 +1537,6 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
         retract($recommendation);
     end
 
-
-// Rule unit: X.5
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
@@ -1547,6 +1548,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512
@@ -1580,6 +1592,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
+
 // Rule unit: X.9
 rule "X.9.0: Merge mostly contained signatures"
     when
@@ -1590,6 +1603,7 @@ rule "X.9.0: Merge mostly contained signatures"
         $signature.addEngine(LayoutEngine.AI);
     end
 
+
 // Rule unit: X.10
 rule "X.10.0: remove false positives of ai"
     when

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl

@@ -34,11 +34,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
 
 global ComponentCreationService componentCreationService
 
-/**
-The imports, globals, queries and rules from this file are required for any component rule file.
-Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
-*/
-
 //------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl

@@ -61,12 +61,6 @@ global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
 
-
-/**
-The imports, globals, queries and rules from this file are required for any entity rule file.
-Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
-*/
-
 //------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"
@@ -75,7 +69,7 @@ query "getFileAttributes"
 
 //------------------------------------ Local dictionary search rules ------------------------------------
 
-// Rule unit: LocalDictionarySearch.0
+// Rule unit: LDS.0
 rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999

redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml

@@ -4,4 +4,5 @@
         xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
     <include file="/mongo/changelog/tenant/1-initial-database.changelog.xml"/>
     <include file="/mongo/changelog/tenant/2-create-indices-for-entries.xml"/>
+    <include file="/mongo/changelog/tenant/7-add-entity-log-value-index.xml"/>
 </databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/7-add-entity-log-value-index.xml

@@ -0,0 +1,23 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="createIndexForEntityLogIdAndValue" author="corina">
+
+        <ext:createIndex collectionName="entity-log-entries">
+            <ext:keys>
+                {
+                "entityLogId": 1,
+                "value": 1,
+                }
+            </ext:keys>
+            <ext:options>
+                {name: "entityLogId_value_index"}
+            </ext:options>
+        </ext:createIndex>
+
+    </changeSet>
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -14,6 +14,7 @@ import java.util.UUID;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -326,6 +327,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
     @Test
     @SneakyThrows
+    @Disabled
     void testNerEntitiesAfterReanalysis() {
 
         String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -15,6 +15,7 @@ import java.util.List;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -134,6 +135,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
 
     @Test
     @SneakyThrows
+    @Disabled
     public void testBuildTextBlockPerformance() {
 
         int n = 10000;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -126,7 +127,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
         assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
         assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
-        assertEquals(unprocessedManualEntity.getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
+        assertThat(unprocessedManualEntity.getSection()).contains("Paragraph: A9396G containing 960 g/L");
         assertEquals(unprocessedManualEntity.getPositions()
                              .get(0).x(), 355.53775f);
         assertEquals(unprocessedManualEntity.getPositions()
@@ -173,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
-        assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
+        assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
         assertEquals(unprocessedManualEntities.get(0).getPositions()
                              .get(0).x(), positions.get(0).getTopLeftX());
         assertEquals(unprocessedManualEntities.get(0).getPositions()
@@ -256,7 +257,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertTrue(resizedAoel.isPresent());
         assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
         assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
-        assertEquals(resizedAoel.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
+        assertThat(resizedAoel.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
         assertEquals(resizedAoel.get().getPositions()
                              .get(0).x(), positions.get(0).getTopLeftX());
         assertEquals(resizedAoel.get().getPositions()
@@ -272,7 +273,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertTrue(cormsResized.isPresent());
         assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
         assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
-        assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
+        assertThat(cormsResized.get().getSection()).contains("Paragraph: The Co-RMS indicated the");
         assertEquals(cormsResized.get().getPositions()
                              .get(0).x(), positions2.get(0).getTopLeftX());
         assertEquals(cormsResized.get().getPositions()
@@ -288,7 +289,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertTrue(a9Resized.isPresent());
         assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
         assertEquals(a9Resized.get().getTextBefore(), "data for S");
-        assertEquals(a9Resized.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
+        assertThat(a9Resized.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
         assertEquals(a9Resized.get().getPositions()
                              .get(0).x(), positions3.get(0).getTopLeftX());
         assertEquals(a9Resized.get().getPositions()
@@ -338,7 +339,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
-        assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
+        assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
         assertEquals(unprocessedManualEntities.get(0).getPositions()
                              .get(0).x(), positions.get(0).getTopLeftX());
         assertEquals(unprocessedManualEntities.get(0).getPositions()
@@ -388,7 +389,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
-        assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
+        assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: With respect to the");
         assertEquals(unprocessedManualEntities.get(0).getPositions()
                              .get(0).x(), positions.get(0).getTopLeftX());
         assertEquals(unprocessedManualEntities.get(0).getPositions()

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -273,6 +272,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
         $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
+
 // Rule unit: CBI.16
 rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
     when
@@ -874,7 +874,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1291,7 +1291,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -1351,6 +1350,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -1430,7 +1440,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -340,7 +339,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     end
 
 
-// Rule unit: CBI.12 - table rules remains
+// Rule unit: CBI.12
 rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
@@ -404,8 +403,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
           .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
-
-//from CBI.3.3
 rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@@ -422,8 +419,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
             });
     end
 
-
-//from CBI.3.1
 rule "CBI.12.5: Redacted because table row contains a vertebrate"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@@ -440,7 +435,6 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
             });
     end
 
-
 rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'"
     salience -1
     when
@@ -535,7 +529,7 @@ rule "CBI.12.15: Redacted because table row contains a vertebrate, a no_redactio
     end
 
 
-// Rule unit: CBI.13 - section rules
+// Rule unit: CBI.13
 rule "CBI.13.0: Ignore CBI Address recommendations"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -545,7 +539,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
         retract($entity)
     end
 
-// from CBI.3.0
 rule "CBI.13.1: Redacted because Section contains a vertebrate"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -561,7 +554,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
             });
     end
 
-//from CBI.3.2
 rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
     when
         $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -570,8 +562,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
             .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
     end
 
-
-// from CBI.4.0
 rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
@@ -589,8 +579,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
             });
     end
 
-
-// from CBI.5.0
 rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
@@ -612,7 +600,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
             });
     end
 
-// From CBI.8.0
 rule "CBI.13.5: Redacted because Section contains must_redact entity"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -628,6 +615,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
             });
     end
 
+
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
@@ -742,6 +730,7 @@ rule "CBI.16.3: Do not redact PII if published information found in same table r
         $pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
     end
 
+
 // Rule unit: CBI.17
 rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
     when
@@ -847,7 +836,6 @@ rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 
 
 // Rule unit: CBI.21
-// from CBI.6
 rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section"
     when
         $section: Section(!hasTables(),
@@ -882,7 +870,6 @@ rule "CBI.21.1: Do not redact Names and Addresses if published_information is fo
             });
     end
 
-
 rule "CBI.21.2: Redact short Authors section (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -1397,7 +1384,6 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
     end
 
 
-// UPDATED WITH LIMIT
 // Rule unit: PII.9
 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
     when
@@ -1444,7 +1430,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
         .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
      end
 
-
 rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -1458,7 +1443,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1475,8 +1460,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-
-// Rule unit: PII.12
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -1591,7 +1574,6 @@ rule "ETC.3.3: Redact logos"
         $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
     end
 
-// from preGFL Knoell
 rule "ETC.3.4: Skip logos"
     when
         $logo: Image(imageType == ImageType.LOGO)
@@ -1759,6 +1741,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
         $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
     end
 
+
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
@@ -1885,7 +1868,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -2075,7 +2058,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -2135,6 +2117,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -2214,7 +2207,6 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
         retract($dictionaryEntity);
     end
 
-
 rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
@@ -2225,7 +2217,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -13,12 +13,14 @@ import java.util.stream.Stream;
 import java.util.Optional;
 
 import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
+import com.iqser.red.service.redaction.v1.server.model.document.*;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@@ -30,8 +32,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
@@ -47,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1419,7 +1426,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -1479,6 +1485,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -139,6 +138,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
         $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
+
 // Rule unit: CBI.1
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
@@ -439,7 +439,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
         .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-
 rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -599,7 +598,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
         .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
      end
 
-
 rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -613,7 +611,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -708,6 +706,7 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
         retract($dossierRedaction);
     end
 
+
 // Rule unit: ETC.12
 rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
     when
@@ -725,6 +724,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
         $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
     end
 
+
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
@@ -1018,7 +1018,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -1078,6 +1077,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -1157,7 +1167,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -149,7 +148,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -339,7 +338,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -399,6 +397,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -478,7 +487,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -85,6 +84,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
 
 
 //------------------------------------ CBI rules ------------------------------------
+
 // Rule unit: CBI.0
 rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
     agenda-group "LOCAL_DICTIONARY_ADDS"
@@ -177,7 +177,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     end
 
 
-// Rule unit: CBI.12 - table rules remains
+// Rule unit: CBI.12
 rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
@@ -241,8 +241,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
           .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
-
-//from CBI.3.3
 rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@@ -259,8 +257,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
             });
     end
 
-
-//from CBI.3.1
 rule "CBI.12.5: Redacted because table row contains a vertebrate"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@@ -278,8 +274,7 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
     end
 
 
-// Rule unit: CBI.13 - section rules
-// from CBI.3.0
+// Rule unit: CBI.13
 rule "CBI.13.1: Redacted because Section contains a vertebrate"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -295,7 +290,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
             });
     end
 
-//from CBI.3.2
 rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
     when
         $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -304,8 +298,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
             .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
     end
 
-
-// from CBI.4.0
 rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
@@ -323,8 +315,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
             });
     end
 
-
-// from CBI.5.0
 rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
@@ -346,7 +336,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
             });
     end
 
-// From CBI.8.0
 rule "CBI.13.5: Redacted because Section contains must_redact entity"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -362,6 +351,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
             });
     end
 
+
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
@@ -854,7 +844,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1133,7 +1123,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1323,7 +1313,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -1383,6 +1372,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -1462,7 +1462,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -397,7 +396,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -457,6 +455,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -536,7 +545,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -489,7 +488,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -549,6 +547,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -639,7 +648,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -389,6 +388,15 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
 
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
@@ -401,19 +409,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
-// Rule unit: X.2
-rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
-    salience 64
-    when
-        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
-    then
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
-        retract($entity)
-    end
-
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -473,6 +468,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -563,7 +569,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta

@@ -1 +1 @@
-Subproject commit 0da08b1d9d1bc815a3fb51aa9638eafea2cf02d5
+Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7

redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl

@@ -4,7 +4,13 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
 
 global RulesLogger logger
 
+//------------------------------------ queries ------------------------------------
 
+
+
+//------------------------------------ LOG rules ------------------------------------
+
+// Rule unit: LOG.0
 rule "LOG.0.0: Test log info"
 salience 1
     when
@@ -27,4 +33,4 @@ rule "LOG.0.2: Test log error"
     then
         String result = null;
         result.toString();
-    end
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@@ -420,7 +421,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
         .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-
 rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -593,7 +593,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -901,7 +901,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         retract($entity)
     end
 
-
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -961,6 +960,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
@@ -1040,7 +1050,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
         retract($nonManualEntity);
     end
 
-
 rule "X.11.2: Remove non manual entity which are equal to manual entity"
     salience 70
     when

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -4,13 +4,11 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
-import java.util.Map;
 
 import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
 import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
 import com.knecon.fforesight.utility.rules.management.models.BasicRule;
 import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
-import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
 import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
 
 import lombok.SneakyThrows;
@@ -23,7 +21,6 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class RuleFileMigrator {
 
-
     @SneakyThrows
     public void migrateFile(File ruleFile) {
 
@@ -53,21 +50,12 @@ public class RuleFileMigrator {
 
     private static void replaceRuleIdentifiers(RuleFileBluePrint combinedBluePrint, RuleFileBluePrint ruleFileBluePrint) {
 
-        Map<String, String> identifierReplaceMap = Map.of("CBI.7.0", "CBI.16.0", "CBI.7.1", "CBI.16.1", "CBI.7.2", "CBI.16.2", "CBI.7.3", "CBI.16.3");
-        for (String identifier : identifierReplaceMap.keySet()) {
-            RuleIdentifier ruleId = RuleIdentifier.fromString(identifier);
-            RuleIdentifier otherRuleId = RuleIdentifier.fromString(identifierReplaceMap.get(identifier));
-
-            List<BasicRule> rulesToAdd = combinedBluePrint.findRuleByIdentifier(otherRuleId);
-            List<BasicRule> otherRulesToAdd = combinedBluePrint.findRuleByIdentifier(ruleId);
-            boolean removeRules = ruleFileBluePrint.removeRule(ruleId);
-            boolean removeOtherRules = ruleFileBluePrint.removeRule(otherRuleId);
-            if (removeRules) {
-                rulesToAdd.forEach(ruleFileBluePrint::addRule);
-            }
-            if (removeOtherRules) {
-                otherRulesToAdd.forEach(ruleFileBluePrint::addRule);
+        for (BasicRule rule : combinedBluePrint.getAllRules()) {
+            if (ruleFileBluePrint.findRuleByIdentifier(rule.identifier()).isEmpty()) {
+                continue;
             }
+            ruleFileBluePrint.removeRule(rule.identifier());
+            ruleFileBluePrint.addRule(rule);
         }
     }
 

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@@ -35,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@@ -55,8 +57,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1459,7 +1459,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -2147,6 +2147,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.6
 rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -13,12 +13,14 @@ import java.util.stream.Stream;
 import java.util.Optional;
 
 import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
+import com.iqser.red.service.redaction.v1.server.model.document.*;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@@ -30,8 +32,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
@@ -47,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1547,6 +1554,17 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !removed())
+        $recommendation: TextEntity(containedBy($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512

redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java

@@ -25,12 +25,10 @@ public class RuleFileMigrationTest {
 
     // Put your redaction service drools paths and dossier-templates paths both RM and DM here
     static final List<String> ruleFileDirs = List.of(
-            //"/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            //                                 "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2"
-            "/Users/maverickstuder/Documents/PM"
-
-            );
-
+            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools"
+//            "/home/kschuettler/iqser/redaction/dossier-templates-v2",
+//            "/home/kschuettler/iqser/fforesight/dossier-templates-v2"
+    );
 
 
     @Test