redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -47,7 +47,7 @@ public class TextEntity implements IEntity {
 
     TextRange textRange;
     @Builder.Default
-    Set<TextRange> duplicateTextRanges = new HashSet<>();
+    List<TextRange> duplicateTextRanges = new ArrayList<>();
     String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
     final EntityType entityType;
 
@@ -215,20 +215,22 @@ public class TextEntity implements IEntity {
         return textEntity.contains(this);
     }
 
+
     public boolean contains(TextEntity textEntity) {
 
         if (this.textRange.contains(textEntity.getTextRange())) {
             return true;
         }
 
-        Set<TextRange> textEntityDuplicateRanges = textEntity.getDuplicateTextRanges();
+        List<TextRange> textEntityDuplicateRanges = textEntity.getDuplicateTextRanges();
-
+        // use optimized indexed loops for extra performance boost
-        for (TextRange duplicateTextRange : this.duplicateTextRanges) {
+        for (int i = 0, duplicateTextRangesSize = duplicateTextRanges.size(); i < duplicateTextRangesSize; i++) {
+            TextRange duplicateTextRange = duplicateTextRanges.get(i);
             if (duplicateTextRange.contains(textEntity.getTextRange())) {
                 return true;
             }
-
+            for (int j = 0, textEntityDuplicateRangesSize = textEntityDuplicateRanges.size(); j < textEntityDuplicateRangesSize; j++) {
-            for (TextRange otherRange : textEntityDuplicateRanges) {
+                TextRange otherRange = textEntityDuplicateRanges.get(j);
                 if (duplicateTextRange.contains(otherRange)) {
                     return true;
                 }
@@ -239,7 +241,6 @@ public class TextEntity implements IEntity {
     }
 
 
-
     public boolean intersects(TextEntity textEntity) {
 
         return this.textRange.intersects(textEntity.getTextRange()) //

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -155,7 +155,7 @@ public class ManualChangesApplicationService {
         entityToBeResized.setTextRange(closestEntity.getTextRange());
         entityToBeResized.setTextAfter(closestEntity.getTextAfter());
         entityToBeResized.setTextBefore(closestEntity.getTextBefore());
-        entityToBeResized.setDuplicateTextRanges(new HashSet<>(closestEntity.getDuplicateTextRanges()));
+        entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
         entityToBeResized.setValue(closestEntity.getValue());
         entityToBeResized.setPages(newIntersectingPages);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -152,7 +152,7 @@ public class EntityFromPrecursorCreationService {
         }
         correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
         correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
-        correctEntity.setDuplicateTextRanges(new HashSet<>(closestEntity.getDuplicateTextRanges()));
+        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
         correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
 
         correctEntity.setValue(closestEntity.getValue());

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -25,8 +25,7 @@ fforesight:
     ignored-endpoints: [ '/redaction-gateway-v1', '/actuator/health/**',"/api/rules-logging/rulesocket","/api/rules-logging/rulesocket/**", '/redaction-gateway-v1/async/download/with-ott/**',
                        '/internal-api/**', '/redaction-gateway-v1/docs/swagger-ui', '/rules/test',
                        '/redaction-gateway-v1/docs/**','/redaction-gateway-v1/docs',
-                       '/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui',
+                       '/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui' ]
-                       '/actuator/prometheus' ]
 
 spring:
   application:

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl

@@ -1584,23 +1584,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -261,6 +261,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Disabled
     @Test
     public void testDoseMortalityExtraction() {
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -1315,7 +1315,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -1385,23 +1384,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -2079,7 +2079,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -2149,23 +2148,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -1451,7 +1451,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -1521,23 +1520,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -1042,7 +1042,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -1112,23 +1111,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -363,7 +363,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -433,23 +432,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1408,23 +1408,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -423,7 +423,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -493,23 +492,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -513,7 +513,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -583,23 +582,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -396,6 +396,38 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
 end
 
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $containment: Containment(
+            $container: container,
+            $contained: contained,
+            $container.entityType == EntityType.FALSE_POSITIVE,
+            $container.active(),
+            $contained.entityType == EntityType.ENTITY,
+            $contained.type() == $container.type(),
+            !$contained.hasManualChanges()
+        )
+    then
+        $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+end
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $containment: Containment(
+            $container: container,
+            $contained: contained,
+            $container.entityType == EntityType.FALSE_POSITIVE,
+            $container.active(),
+            $contained.entityType == EntityType.HINT,
+            $contained.type() == $container.type(),
+            !$contained.hasManualChanges()
+        )
+    then
+        $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE");
+end
+
 rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
     salience 64
     when
@@ -413,7 +445,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
 end
 
 
-// Rule unit: X.3
 rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
     salience 64
     when
@@ -483,23 +514,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -995,23 +995,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -2187,23 +2187,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.6
 rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
     salience 32

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -1589,23 +1589,6 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 end
 
 
-rule "X.5.2: Remove Entity of type RECOMMENDATION when contained by ENTITY of same type"
-    salience 256
-    when
-        $intersection: Containment(
-            $container: container,
-            $contained: contained,
-            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
-            !$container.removed(),
-            $contained.entityType == EntityType.RECOMMENDATION,
-            $container.type() == $contained.type(),
-            !$contained.hasManualChanges()
-        )
-    then
-        $contained.remove("X.5.2", "remove Entity of type RECOMMENDATION when contained by ENTITY of same type");
-    end
-
-
 // Rule unit: X.7
 rule "X.7.0: Remove all images"
     salience 512