test publish stuff

RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature

Closes RED-8590

See merge request redactmanager/redaction-service!293

.gitlab-ci.yml

@@ -21,3 +21,16 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+
+pages:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
+    - mkdir public
+    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+  artifacts:
+    paths:
+      - public

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.338.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DroolsSyntaxDeprecatedWarnings {
+
+    Integer line;
+    Integer column;
+    String message;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -19,6 +19,8 @@ public class DroolsSyntaxValidation {
 
     @Builder.Default
     List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
+    @Builder.Default
+    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
 
 
     public void addErrorMessage(int line, int column, String message) {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,15 +1,11 @@
 package com.iqser.red.service.redaction.v1.model;
 
-import java.util.Collections;
-import java.util.Set;
-
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.NonNull;
 
 @Data
 @Builder
@@ -17,18 +13,9 @@ import lombok.NonNull;
 @AllArgsConstructor
 public class MigrationRequest {
 
-    @NonNull
     String dossierTemplateId;
-    @NonNull
     String dossierId;
-    @NonNull
     String fileId;
-
-    boolean fileIsApproved;
-    @NonNull
     ManualRedactions manualRedactions;
-    @NonNull
-    @Builder.Default
-    Set<String> entitiesWithComments = Collections.emptySet();
 
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.89.11"
+val layoutParserVersion = "0.94.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.349.26"
+val persistenceServiceVersion = "2.360.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {
@@ -60,6 +60,8 @@ dependencies {
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
 
+    implementation("org.reflections:reflections:0.10.2")
+
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -82,9 +84,6 @@ tasks.test {
     maxHeapSize = "1024m"
 }
 
-
-
-
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -111,3 +110,40 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         tags.set(listOf(dockerTag))
     }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -46,10 +46,18 @@ public class Application {
         return new TimedAspect(registry);
     }
 
+
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
+
+    @Bean(initMethod = "findDeprecatedMethods")
+    public DeprecatedElementsFinder deprecatedElementsFinder() {
+
+        return new DeprecatedElementsFinder();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.ConfigurationBuilder;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DeprecatedElementsFinder {
+
+    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    private Set<Method> deprecatedMethods;
+    @Getter
+    private Map<String, String> deprecatedMethodsSignaturesMap;
+    private Set<Class<?>> deprecatedClasses;
+    @Getter
+    private SearchImplementation classesSearchImplementation;
+    @Getter
+    private SearchImplementation methodsSearchImplementation;
+
+
+    public void findDeprecatedMethods() {
+
+        deprecatedMethods = new HashSet<>();
+        deprecatedClasses = new HashSet<>();
+        deprecatedMethodsSignaturesMap = new HashMap<>();
+
+        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
+                                                          .setExpandSuperTypes(true)
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+
+        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
+
+        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
+
+        if (!deprecatedClasses.isEmpty()) {
+            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
+                                                                                         .collect(Collectors.toSet())));
+            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
+                                                                           .map(Class::getName)
+                                                                           .collect(Collectors.toSet()), false);
+        }
+
+        if (!deprecatedMethods.isEmpty()) {
+            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
+                                                                           .map(Method::getName)
+                                                                           .collect(Collectors.toSet()), false);
+            deprecatedMethodsSignaturesMap = new HashMap<>();
+            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
+                    .collect(groupingBy(Method::getName));
+            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
+                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
+                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
+                                                                       .stream()
+                                                                       .map(this::getMethodSignature)
+                                                                       .collect(Collectors.joining("\n")));
+            }
+
+        }
+
+        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
+    }
+
+
+    public Set<Method> getDeprecatedMethods() {
+
+        if (this.deprecatedMethods == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedMethods;
+    }
+
+
+    public Set<Class<?>> getDeprecatedClasses() {
+
+        if (this.deprecatedClasses == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedClasses;
+    }
+
+    private String getMethodSignature(Method method) {
+
+        String methodName = method.getName();
+        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
+                .map(Type::getTypeName)
+                .collect(Collectors.joining(", "));
+
+        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -30,10 +30,4 @@ public class RedactionServiceSettings {
 
     private int droolsExecutionTimeoutSecs = 300;
 
-
-    public int getDroolsExecutionTimeoutSecs(int numberOfPages) {
-
-        return (int) Math.max(getDroolsExecutionTimeoutSecs(), getDroolsExecutionTimeoutSecs() * ((float) numberOfPages / 1000));
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
@@ -93,16 +92,11 @@ public class LegacyRedactionLogMergeService {
         return redactionLog;
     }
 
-
     public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
 
-        return createManualRedactionWrappers(manualRedactions).stream()
-                .map(ManualRedactionWrapper::getId)
-                .distinct()
-                .count();
+        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
     }
 
-
     private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
 
         List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
@@ -203,12 +197,7 @@ public class LegacyRedactionLogMergeService {
         }
 
         redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization)
-                             .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                             .withChange("type", imageRecategorization.getType())
-                             .withChange("section", imageRecategorization.getSection())
-                             .withChange("legalBasis", imageRecategorization.getLegalBasis())
-                             .withChange("value", imageRecategorization.getValue()));
+                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
     }
 
 
@@ -380,17 +369,13 @@ public class LegacyRedactionLogMergeService {
     @SuppressWarnings("PMD.UselessParentheses")
     private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
 
-        if (manualRedactionEntry.getDictionaryEntryType() != null //
-            && (manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE) //
-                || manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_RECOMMENDATION))) {
+        if (!manualRedactionEntry.isApproved()) {
             return false;
         }
 
-        if (manualRedactionEntry.getProcessedDate() == null) {
-            return false;
-        }
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary());
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
+                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
+                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -1,9 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.migration;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
@@ -70,21 +70,13 @@ public class MigrationMapper {
 
     public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
 
-        Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines = new HashSet<>();
-
-        if (entry.isImported()) {
-            engines.add(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.IMPORTED);
-        }
-
         if (entry.getEngines() == null) {
-            return engines;
+            return Collections.emptySet();
         }
-        entry.getEngines()
+        return entry.getEngines()
                 .stream()
                 .map(MigrationMapper::toEntityLogEngine)
-                .forEach(engines::add);
-
-        return engines;
+                .collect(Collectors.toSet());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -57,7 +57,7 @@ public class MigrationMessageReceiver {
 
         if (redactionLog.getAnalysisVersion() == 0) {
             redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else {
+        } else if (migrationRequest.getManualRedactions() != null) {
             redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
                                                                                                       migrationRequest.getManualRedactions(),
                                                                                                       migrationRequest.getDossierTemplateId());
@@ -67,9 +67,7 @@ public class MigrationMessageReceiver {
                                                                                               document,
                                                                                               migrationRequest.getDossierTemplateId(),
                                                                                               migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId(),
-                                                                                              migrationRequest.getEntitiesWithComments(),
-                                                                                              migrationRequest.isFileIsApproved());
+                                                                                              migrationRequest.getFileId());
 
         log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -8,7 +8,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -20,9 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@@ -37,6 +34,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
@@ -56,17 +54,12 @@ public class RedactionLogToEntityLogMigrationService {
 
     private static final double MATCH_THRESHOLD = 10;
     EntityFindingUtility entityFindingUtility;
+    EntityEnrichmentService entityEnrichmentService;
     DictionaryService dictionaryService;
     ManualChangesApplicationService manualChangesApplicationService;
 
 
-    public MigratedEntityLog migrate(RedactionLog redactionLog,
-                                     Document document,
-                                     String dossierTemplateId,
-                                     ManualRedactions manualRedactions,
-                                     String fileId,
-                                     Set<String> entitiesWithComments,
-                                     boolean fileIsApproved) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
 
         log.info("Migrating entities for file {}", fileId);
         List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
@@ -74,8 +67,8 @@ public class RedactionLogToEntityLogMigrationService {
         MigratedIds migratedIds = entitiesToMigrate.stream()
                 .collect(new MigratedIdsCollector());
 
+        applyManualChanges(entitiesToMigrate, manualRedactions);
         log.info("applying manual changes to migrated entities for file {}", fileId);
-        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -96,7 +89,7 @@ public class RedactionLogToEntityLogMigrationService {
                                             .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
                                             .toList());
 
-        if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
+        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
             String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
                                            redactionLog.getRedactionLogEntry().size(),
                                            entityLog.getEntityLogEntry().size());
@@ -104,14 +97,8 @@ public class RedactionLogToEntityLogMigrationService {
             throw new AssertionError(message);
         }
 
-        Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
-                .stream()
-                .filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
-                .map(BaseAnnotation::getAnnotationId)
-                .collect(Collectors.toSet());
-
         MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
+                .filter(MigrationEntity::hasManualChangesOrComments)
                 .filter(m -> !m.getOldId().equals(m.getNewId()))
                 .collect(new MigratedIdsCollector());
 
@@ -126,29 +113,20 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {
+    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return;
         }
-        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;
 
-        if (fileIsApproved) {
-            manualChangesPerAnnotationId = manualRedactions.buildAll()
-                    .stream()
-                    .filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
-                                            // unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
-                                            || (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
-                                                && (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
-                    .map(this::convertPendingDictChangesToLocal)
-                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-        } else {
-            manualChangesPerAnnotationId = manualRedactions.buildAll()
-                    .stream()
-                    .filter(manualChange -> manualChange.getProcessedDate() != null)
-                    .filter(BaseAnnotation::isLocal)
-                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-        }
+        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
+                                                                                   manualRedactions.getEntriesToAdd(),
+                                                                                   manualRedactions.getForceRedactions(),
+                                                                                   manualRedactions.getResizeRedactions(),
+                                                                                   manualRedactions.getLegalBasisChanges(),
+                                                                                   manualRedactions.getRecategorizations())
+                .flatMap(Collection::stream)
+                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
 
         entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
                                                                                                                                   Collections.emptyList()),
@@ -157,40 +135,15 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {
+    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
 
-        if (baseAnnotation.getProcessedDate() != null) {
-            return baseAnnotation;
-        }
-
-        if (baseAnnotation.isLocal()) {
-            return baseAnnotation;
-        }
-
-        if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
-            manualResizeRedaction.setAddToAllDossiers(false);
-            manualResizeRedaction.setUpdateDictionary(false);
-        } else if (baseAnnotation instanceof IdRemoval idRemoval) {
-            idRemoval.setRemoveFromAllDossiers(false);
-            idRemoval.setRemoveFromDictionary(false);
-        }
-
-        return baseAnnotation;
-    }
-
-
-    private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
-
-        return redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
-                .count();
+        return redactionLog.getRedactionLogEntry().size();
     }
 
 
     private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
         List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
         return Stream.of(textMigrationEntities.stream(), images.stream())
                 .flatMap(Function.identity())
@@ -204,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
 
         List<Image> images = document.streamAllImages()
                 .collect(Collectors.toList());
@@ -251,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
             } else {
                 closestImage.skip(ruleIdentifier, reason);
             }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
         }
         return migrationEntities;
     }
@@ -297,8 +250,7 @@ public class RedactionLogToEntityLogMigrationService {
         List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
                 .toList();
 
         List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
@@ -335,20 +287,4 @@ public class RedactionLogToEntityLogMigrationService {
         return entitiesToMigrate;
     }
 
-
-    private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
-
-        var pages = redactionLogEntry.getPositions()
-                .stream()
-                .map(Rectangle::getPage)
-                .collect(Collectors.toSet());
-
-        for (int page : pages) {
-            if (page > numberOfPages) {
-                return false;
-            }
-        }
-        return true;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -4,11 +4,9 @@ import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreator
 import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
 
 import java.awt.geom.Rectangle2D;
-import java.time.OffsetDateTime;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -20,10 +18,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
@@ -32,8 +28,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
@@ -52,8 +46,6 @@ public final class MigrationEntity {
 
     private final PrecursorEntity precursorEntity;
     private final RedactionLogEntry redactionLogEntry;
-    private final DictionaryService dictionaryService;
-    private final String dossierTemplateId;
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
@@ -63,10 +55,10 @@ public final class MigrationEntity {
     List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
 
-        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
         PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
+
         if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
             precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         } else if (redactionLogEntry.lastChangeIsRemoved()) {
@@ -81,32 +73,13 @@ public final class MigrationEntity {
             precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         }
 
-        return MigrationEntity.builder()
-                .precursorEntity(precursorEntity)
-                .redactionLogEntry(redactionLogEntry)
-                .oldId(redactionLogEntry.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
+        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
     }
 
 
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
-                                                        Image image,
-                                                        String fileId,
-                                                        DictionaryService dictionaryService,
-                                                        String dossierTemplateId) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
 
-        return MigrationEntity.builder()
-                .redactionLogEntry(redactionLogImage)
-                .migratedEntity(image)
-                .oldId(redactionLogImage.getId())
-                .newId(image.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
+        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
     }
 
 
@@ -183,6 +156,18 @@ public final class MigrationEntity {
     }
 
 
+    private static EntryType getEntryType(EntityType entityType) {
+
+        return switch (entityType) {
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
+            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
+            case RECOMMENDATION -> EntryType.RECOMMENDATION;
+            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+        };
+    }
+
+
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -205,15 +190,14 @@ public final class MigrationEntity {
         entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
         entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
         entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
+        if (redactionLogEntry.getLegalBasis() != null) {
+            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        }
 
         if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
             entityLogEntry.setState(EntryState.IGNORED);
         }
 
-        if (redactionLogEntry.isImported() && redactionLogEntry.getValue() == null) {
-            entityLogEntry.setValue("Imported Redaction");
-        }
-
         return entityLogEntry;
     }
 
@@ -241,15 +225,13 @@ public final class MigrationEntity {
 
     public EntityLogEntry createEntityLogEntry(Image image) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
         return EntityLogEntry.builder()
                 .id(image.getId())
-                .value(image.getValue())
-                .type(imageType)
+                .value(image.value())
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(false)
                 .positions(positions)
@@ -261,7 +243,7 @@ public final class MigrationEntity {
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
                 .state(buildEntryState(image))
-                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
@@ -272,8 +254,7 @@ public final class MigrationEntity {
         return EntityLogEntry.builder()
                 .id(precursorEntity.getId())
                 .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(precursorEntity.legalBasis())
                 .value(precursorEntity.value())
                 .type(precursorEntity.type())
                 .state(buildEntryState(precursorEntity))
@@ -307,8 +288,7 @@ public final class MigrationEntity {
                 .id(entity.getId())
                 .positions(rectanglesPerLine)
                 .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue()
                                .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                 .type(entity.type())
@@ -351,11 +331,11 @@ public final class MigrationEntity {
     }
 
 
-    public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
+    public boolean hasManualChangesOrComments() {
 
         return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
                !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
+               || hasManualChanges();
     }
 
 
@@ -370,11 +350,17 @@ public final class MigrationEntity {
         manualChanges.addAll(manualChangesToApply);
         manualChangesToApply.forEach(manualChange -> {
             if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
-                manualResizeRedaction.setAnnotationId(newId);
-                manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
-            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
-                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
-                migratedEntity.getManualOverwrite().addChange(manualChange);
+                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
+                // Instead, we add an add_locally change to the db.
+                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
+                        .positions(manualResizeRedaction.getPositions())
+                        .annotationId(getNewId())
+                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
+                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
+                        .textAfter(manualResizeRedaction.getTextAfter())
+                        .textBefore(manualResizeRedaction.getTextBefore())
+                        .build();
+                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
             } else {
                 migratedEntity.getManualOverwrite().addChange(manualChange);
             }
@@ -392,25 +378,19 @@ public final class MigrationEntity {
                 .findFirst()
                 .orElse(manualChanges.get(0)).getUser();
 
-        OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
-
         return ManualRedactionEntry.builder()
                 .annotationId(newId)
                 .fileId(fileId)
-                .user(user)
-                .requestDate(requestDate)
                 .type(redactionLogEntry.getType())
                 .value(redactionLogEntry.getValue())
                 .reason(redactionLogEntry.getReason())
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .section(redactionLogEntry.getSection())
-                .rectangle(false)
                 .addToDictionary(false)
                 .addToDossierDictionary(false)
+                .rectangle(false)
                 .positions(buildPositions(migratedEntity))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .dictionaryEntryType(DictionaryEntryType.ENTRY)
+                .user(user)
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.document;
 
 import static java.lang.String.format;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -41,10 +40,7 @@ public class DocumentTree {
 
     public TextBlock buildTextBlock() {
 
-        return allEntriesInOrder().map(Entry::getNode)
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
-                .collect(new TextBlockCollector());
+        return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
     }
 
 
@@ -93,8 +89,8 @@ public class DocumentTree {
         if (treeId.isEmpty()) {
             return root != null;
         }
-        Entry entry = root;
-        for (int id : treeId) {
+        Entry entry = root.children.get(treeId.get(0));
+        for (int id : treeId.subList(1, treeId.size())) {
             if (id >= entry.children.size() || 0 > id) {
                 return false;
             }
@@ -118,78 +114,13 @@ public class DocumentTree {
 
     public Stream<SemanticNode> childNodes(List<Integer> treeId) {
 
-        return getEntryById(treeId).children.stream()
-                .map(Entry::getNode);
-    }
-
-
-    /**
-     * Finds all child nodes of the specified entry, whose nodes textRange intersects the given textRange. It achieves this by finding the first entry, whose textRange contains the start idx of the TextRange using a binary search.
-     * It then iterates over the remaining children adding them to the intersections, until one does not contain the end of the TextRange. All intersected Entries are returned as SemanticNodes.
-     *
-     * @param treeId    the treeId of the Entry whose children shall be checked.
-     * @param textRange The TextRange to find intersecting childNodes for.
-     * @return A list of all SemanticNodes, that are direct children of the specified Entry, whose TextRange intersects the given TextRange
-     */
-    public List<SemanticNode> findIntersectingChildNodes(List<Integer> treeId, TextRange textRange) {
-
-        List<Entry> childEntries = getEntryById(treeId).getChildren();
-        List<SemanticNode> intersectingChildEntries = new LinkedList<>();
-        int startIdx = findFirstIdxOfContainingChildBinarySearch(childEntries, textRange.start());
-        if (startIdx < 0) {
-            return intersectingChildEntries;
-        }
-        for (int i = startIdx; i < childEntries.size(); i++) {
-            if (childEntries.get(i).getNode().getTextRange().start() < textRange.end()) {
-                intersectingChildEntries.add(childEntries.get(i).getNode());
-            } else {
-                break;
-            }
-        }
-        return intersectingChildEntries;
-    }
-
-
-    public Optional<SemanticNode> findFirstContainingChild(List<Integer> treeId, TextRange textRange) {
-
-        List<Entry> childEntries = getEntryById(treeId).getChildren();
-        int startIdx = findFirstIdxOfContainingChildBinarySearch(childEntries, textRange.start());
-        if (startIdx < 0) {
-            return Optional.empty();
-        }
-
-        if (childEntries.get(startIdx).getNode().getTextRange().contains(textRange.end())) {
-            return Optional.of(childEntries.get(startIdx).getNode());
-        }
-
-        return Optional.empty();
-    }
-
-
-    private int findFirstIdxOfContainingChildBinarySearch(List<Entry> childNodes, int start) {
-
-        int low = 0;
-        int high = childNodes.size() - 1;
-        while (low <= high) {
-            int mid = low + (high - low) / 2;
-            TextRange range = childNodes.get(mid).getNode().getTextRange();
-            if (range.start() > start) {
-                high = mid - 1;
-            } else if (range.end() <= start) {
-                low = mid + 1;
-            } else {
-                return mid;
-            }
-        }
-        return -1;
+        return getEntryById(treeId).children.stream().map(Entry::getNode);
     }
 
 
     public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
 
-        return getEntryById(treeId).children.stream()
-                .filter(entry -> entry.node.getType().equals(nodeType))
-                .map(Entry::getNode);
+        return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode);
     }
 
 
@@ -268,32 +199,26 @@ public class DocumentTree {
 
     public Stream<Entry> allEntriesInOrder() {
 
-        return Stream.of(root)
-                .flatMap(DocumentTree::flatten);
+        return Stream.of(root).flatMap(DocumentTree::flatten);
     }
 
 
     public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
 
-        return getEntryById(parentId).children.stream()
-                .flatMap(DocumentTree::flatten);
+        return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten);
     }
 
 
     @Override
     public String toString() {
 
-        return String.join("\n",
-                           allEntriesInOrder().map(Entry::toString)
-                                   .toList());
+        return String.join("\n", allEntriesInOrder().map(Entry::toString).toList());
     }
 
 
     private static Stream<Entry> flatten(Entry entry) {
 
-        return Stream.concat(Stream.of(entry),
-                             entry.children.stream()
-                                     .flatMap(DocumentTree::flatten));
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten));
     }
 
 
@@ -315,7 +240,7 @@ public class DocumentTree {
         List<Integer> treeId;
         SemanticNode node;
         @Builder.Default
-        List<Entry> children = new ArrayList<>();
+        List<Entry> children = new LinkedList<>();
 
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java

@@ -92,18 +92,12 @@ public class TextRange implements Comparable<TextRange> {
 
     public List<TextRange> split(List<Integer> splitIndices) {
 
-        if (splitIndices.stream()
-                .anyMatch(idx -> !this.contains(idx))) {
-            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
-                                                       splitIndices.stream()
-                                                               .filter(idx -> !this.contains(idx))
-                                                               .toList(),
-                                                       this));
+        if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
+            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
         }
         List<TextRange> splitBoundaries = new LinkedList<>();
         int previousIndex = start;
-        for (int i = 0, splitIndicesSize = splitIndices.size(); i < splitIndicesSize; i++) {
-            int splitIndex = splitIndices.get(i);
+        for (int splitIndex : splitIndices) {
 
             // skip split if it would produce a boundary of length 0
             if (splitIndex == previousIndex) {
@@ -119,12 +113,8 @@ public class TextRange implements Comparable<TextRange> {
 
     public static TextRange merge(Collection<TextRange> boundaries) {
 
-        int minStart = boundaries.stream()
-                .mapToInt(TextRange::start)
-                .min().orElseThrow(IllegalArgumentException::new);
-        int maxEnd = boundaries.stream()
-                .mapToInt(TextRange::end)
-                .max().orElseThrow(IllegalArgumentException::new);
+        int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new);
+        int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new);
         return new TextRange(minStart, maxEnd);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -129,12 +129,8 @@ public class ManualChangeOverwrite {
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
-                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
-                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
-                section = recategorization.getSection();
-                value = recategorization.getValue();
                 if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
                     legalBasis = recategorization.getLegalBasis();
                 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,6 +14,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -33,6 +34,8 @@ public class Document implements GenericSemanticNode {
 
     @EqualsAndHashCode.Include
     List<Integer> treeId = Collections.emptyList();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     Set<Page> pages;
     DocumentTree documentTree;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Footer implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Header implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -17,6 +17,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualCha
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -34,12 +36,13 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
 
-    TextBlock leafTextBlock;
-
     ImageType imageType;
     boolean transparent;
     Rectangle2D position;
@@ -50,11 +53,14 @@ public class Image implements GenericSemanticNode, IEntity {
     @Builder.Default
     ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
+    @EqualsAndHashCode.Exclude
     Page page;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
 
@@ -68,7 +74,9 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextBlock getTextBlock() {
 
-        return leafTextBlock;
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -82,21 +90,15 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextRange getTextRange() {
 
-        return leafTextBlock.getTextRange();
-    }
-
-
-    @Override
-    public int length() {
-
-        return getTextRange().length();
+        return GenericSemanticNode.super.getTextRange();
     }
 
 
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(imageType.toString().toLowerCase(Locale.ENGLISH));
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString());
     }
 
 
@@ -128,4 +130,36 @@ public class Image implements GenericSemanticNode, IEntity {
         return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
     }
 
+
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if(bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
+        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+        return (area / calculatedIntersection) > containmentThreshold;
+    }
+
+
+    public int length() {
+
+        return 0;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java

@@ -6,10 +6,22 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER,
+    OTHER {
+        @Override
+        public String toString() {
+
+            return "image";
+        }
+    },
     OCR;
 
 
+    public String toString() {
+
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -10,12 +10,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
 
@@ -37,6 +40,9 @@ public class Section implements GenericSemanticNode {
 
     Map<Page, Rectangle2D> bBoxCache;
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
 
     @Override
     public NodeType getType() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -70,9 +71,7 @@ public interface SemanticNode {
      */
     default Page getFirstPage() {
 
-        return getTextBlock().getPages()
-                .stream()
-                .min(Comparator.comparingInt(Page::getNumber)).orElseThrow();
+        return getTextBlock().getPages().stream().min(Comparator.comparingInt(Page::getNumber)).orElseThrow();
     }
 
 
@@ -98,8 +97,7 @@ public interface SemanticNode {
      */
     default boolean onPage(int pageNumber) {
 
-        return getPages().stream()
-                .anyMatch(page -> page.getNumber() == pageNumber);
+        return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
     }
 
 
@@ -251,9 +249,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> redactionEntity.type().equals(type));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
     }
 
 
@@ -266,10 +262,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> Arrays.stream(types)
-                        .anyMatch(type -> redactionEntity.type().equals(type)));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
     }
 
 
@@ -282,12 +275,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .map(TextEntity::type)
-                .collect(Collectors.toUnmodifiableSet())
-                .containsAll(Arrays.stream(types)
-                                     .toList());
+        return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
     }
 
 
@@ -300,10 +288,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> redactionEntity.type().equals(type))
-                .toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
     }
 
 
@@ -316,10 +301,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(List<String> types) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> redactionEntity.isAnyType(types))
-                .toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
     }
 
 
@@ -332,11 +314,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String... types) {
 
-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types)
-                                                                             .toList()))
-                .toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
     }
 
 
@@ -379,6 +357,13 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
+    Set<LayoutEngine> getEngines();
+
+    default void addEngine(LayoutEngine engine) {
+        getEngines().add(engine);
+    }
+
+
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -388,8 +373,7 @@ public interface SemanticNode {
      */
     default boolean containsAllStrings(String... strings) {
 
-        return Arrays.stream(strings)
-                .allMatch(this::containsString);
+        return Arrays.stream(strings).allMatch(this::containsString);
     }
 
 
@@ -401,8 +385,7 @@ public interface SemanticNode {
      */
     default boolean containsAnyString(String... strings) {
 
-        return Arrays.stream(strings)
-                .anyMatch(this::containsString);
+        return Arrays.stream(strings).anyMatch(this::containsString);
     }
 
 
@@ -414,8 +397,7 @@ public interface SemanticNode {
      */
     default boolean containsAnyString(List<String> strings) {
 
-        return strings.stream()
-                .anyMatch(this::containsString);
+        return strings.stream().anyMatch(this::containsString);
     }
 
 
@@ -439,8 +421,7 @@ public interface SemanticNode {
      */
     default boolean containsAnyStringIgnoreCase(String... strings) {
 
-        return Arrays.stream(strings)
-                .anyMatch(this::containsStringIgnoreCase);
+        return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase);
     }
 
 
@@ -452,8 +433,7 @@ public interface SemanticNode {
      */
     default boolean containsAllStringsIgnoreCase(String... strings) {
 
-        return Arrays.stream(strings)
-                .allMatch(this::containsStringIgnoreCase);
+        return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase);
     }
 
 
@@ -465,9 +445,7 @@ public interface SemanticNode {
      */
     default boolean containsWord(String word) {
 
-        return getTextBlock().getWords()
-                .stream()
-                .anyMatch(s -> s.equals(word));
+        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
     }
 
 
@@ -479,10 +457,7 @@ public interface SemanticNode {
      */
     default boolean containsWordIgnoreCase(String word) {
 
-        return getTextBlock().getWords()
-                .stream()
-                .map(String::toLowerCase)
-                .anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
+        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
     }
 
 
@@ -494,10 +469,7 @@ public interface SemanticNode {
      */
     default boolean containsAnyWord(String... words) {
 
-        return Arrays.stream(words)
-                .anyMatch(word -> getTextBlock().getWords()
-                        .stream()
-                        .anyMatch(word::equals));
+        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
     }
 
 
@@ -509,12 +481,7 @@ public interface SemanticNode {
      */
     default boolean containsAnyWordIgnoreCase(String... words) {
 
-        return Arrays.stream(words)
-                .map(String::toLowerCase)
-                .anyMatch(word -> getTextBlock().getWords()
-                        .stream()
-                        .map(String::toLowerCase)
-                        .anyMatch(word::equals));
+        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
     }
 
 
@@ -526,10 +493,7 @@ public interface SemanticNode {
      */
     default boolean containsAllWords(String... words) {
 
-        return Arrays.stream(words)
-                .allMatch(word -> getTextBlock().getWords()
-                        .stream()
-                        .anyMatch(word::equals));
+        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
     }
 
 
@@ -541,12 +505,7 @@ public interface SemanticNode {
      */
     default boolean containsAllWordsIgnoreCase(String... words) {
 
-        return Arrays.stream(words)
-                .map(String::toLowerCase)
-                .allMatch(word -> getTextBlock().getWords()
-                        .stream()
-                        .map(String::toLowerCase)
-                        .anyMatch(word::equals));
+        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
     }
 
 
@@ -586,11 +545,7 @@ public interface SemanticNode {
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
 
-        return getBBox().entrySet()
-                .stream()
-                .filter(entry -> entry.getKey().getNumber() == pageNumber)
-                .map(Map.Entry::getValue)
-                .anyMatch(rect -> rect.intersects(x, y, w, h));
+        return getBBox().entrySet().stream().filter(entry -> entry.getKey().getNumber() == pageNumber).map(Map.Entry::getValue).anyMatch(rect -> rect.intersects(x, y, w, h));
     }
 
 
@@ -607,9 +562,8 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
-
             textEntity.addIntersectingNode(this);
-            getDocumentTree().findIntersectingChildNodes(getTreeId(), textEntity.getTextRange())
+            streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));
         }
     }
@@ -644,8 +598,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodes() {
 
-        return getDocumentTree().allSubEntriesInOrder(getTreeId())
-                .map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode);
     }
 
 
@@ -656,9 +609,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType) {
 
-        return getDocumentTree().allSubEntriesInOrder(getTreeId())
-                .filter(entry -> entry.getType().equals(nodeType))
-                .map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode);
     }
 
 
@@ -697,8 +648,7 @@ public interface SemanticNode {
         if (isLeaf()) {
             return getTextBlock().getPositionsPerPage(textRange);
         }
-        Optional<SemanticNode> containingChildNode = getDocumentTree().findFirstContainingChild(getTreeId(), textRange);
-
+        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
         if (containingChildNode.isEmpty()) {
             return getTextBlock().getPositionsPerPage(textRange);
         }
@@ -748,12 +698,8 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromChildren() {
 
         Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox)
-                .toList();
-        Set<Page> pages = childrenBBoxes.stream()
-                .flatMap(map -> map.keySet()
-                        .stream())
-                .collect(Collectors.toSet());
+        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
+        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
         for (Page page : pages) {
             Rectangle2D bBoxOnPage = childrenBBoxes.stream()
                     .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
@@ -771,9 +717,7 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
 
         Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks()
-                .stream()
-                .collect(Collectors.groupingBy(AtomicTextBlock::getPage));
+        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
         atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
         return bBoxPerPage;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -32,6 +33,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -17,7 +17,7 @@ public class MessageReceiver {
 
 
     @RabbitHandler
-    @RabbitListener(queues = REDACTION_QUEUE, concurrency = "1")
+    @RabbitListener(queues = REDACTION_QUEUE)
     public void receiveAnalyzeRequest(Message message) {
 
         redactionMessageReceiver.receiveAnalyzeRequest(message, false);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -17,7 +17,7 @@ public class PriorityMessageReceiver {
 
 
     @RabbitHandler
-    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE, concurrency = "1")
+    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
     public void receiveAnalyzeRequest(Message message) {
 
         redactionMessageReceiver.receiveAnalyzeRequest(message, true);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -12,7 +13,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -38,11 +41,9 @@ public class EntityChangeLogService {
         boolean hasChanges = false;
 
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-
             Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
                     .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                     .findAny();
-
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -50,18 +51,14 @@ public class EntityChangeLogService {
             }
 
             EntityLogEntry previousEntity = optionalPreviousEntity.get();
-
             entityLogEntry.getChanges().addAll(previousEntity.getChanges());
-
             if (!previousEntity.getState().equals(entityLogEntry.getState())) {
                 hasChanges = true;
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
                 entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
             }
         }
-
         addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
-
         return hasChanges;
     }
 
@@ -75,20 +72,36 @@ public class EntityChangeLogService {
         Set<String> existingIds = newEntityLogEntries.stream()
                 .map(EntityLogEntry::getId)
                 .collect(Collectors.toSet());
-
         List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                 .filter(entry -> !existingIds.contains(entry.getId()))
+                .collect(Collectors.toList());
+        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
+                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
                 .toList();
-
-        removedEntries.stream()
-                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
-                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
-                .forEach(entry -> entry.setState(EntryState.REMOVED));
-
+        previousEntityLogEntries.removeAll(removedDossierRedaction);
+        removedEntries.removeAll(removedDossierRedaction);
+        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
+        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
         newEntityLogEntries.addAll(removedEntries);
     }
 
 
+    private void addManualChangeForDictionaryRemovals(EntityLogEntry entry, ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null || manualRedactions.getIdsToRemove().isEmpty()) {
+            return;
+        }
+
+        manualRedactions.getIdsToRemove()
+                .stream()
+                .filter(IdRemoval::isRemoveFromDictionary)//
+                .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
+                .findFirst()//
+                .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
+    }
+
+
     private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
         if (state.equals(previousState)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -186,12 +186,11 @@ public class EntityLogCreatorService {
 
     private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(image.getId())
-                .value(image.getValue())
-                .type(imageType)
+                .value(image.value())
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -30,12 +30,8 @@ public class ManualChangeFactory {
     public ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
 
         ManualChange manualChange = ManualChange.from(baseAnnotation);
-        if (baseAnnotation instanceof ManualRecategorization recategorization) {
-            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                    .withChange("type", recategorization.getType())
-                    .withChange("section", recategorization.getSection())
-                    .withChange("legalBasis", recategorization.getLegalBasis())
-                    .withChange("value", recategorization.getValue());
+        if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
+            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
             manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
         } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -1,29 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -40,9 +40,7 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
-                                        .map(DocumentGraphMapper::buildPage)
-                                        .toList());
+        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());
 
         context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
 
@@ -57,12 +55,10 @@ public class DocumentGraphMapper {
 
     private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
 
-        List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size());
+        List<DocumentTree.Entry> newEntries = new LinkedList<>();
         for (DocumentStructure.EntryData entryData : entries) {
 
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
-                    .map(pageNumber -> getPage(pageNumber, context))
-                    .toList();
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
 
             SemanticNode node = switch (entryData.getType()) {
                 case SECTION -> buildSection(context);
@@ -80,8 +76,8 @@ public class DocumentGraphMapper {
                 TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
             }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
-                    .toList();
+            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {
@@ -154,18 +150,16 @@ public class DocumentGraphMapper {
 
     private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return Arrays.stream(atomicTextBlockIds)
-                .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
-                .collect(new TextBlockCollector());
+        return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector());
     }
 
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
         return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
-                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
-                                                       parent,
-                                                       getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
+                parent,
+                getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
     }
 
 
@@ -179,7 +173,8 @@ public class DocumentGraphMapper {
 
         return context.pageData.stream()
                 .filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
-                .findFirst().orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
     }
 
 
@@ -195,10 +190,8 @@ public class DocumentGraphMapper {
 
             this.documentTree = documentTree;
             this.pageData = new LinkedList<>();
-            this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
-                    .toList();
-            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
-                    .toList();
+            this.documentTextData = Arrays.stream(documentData.getDocumentTextData()).toList();
+            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData()).toList();
 
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1,18 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange;
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
+import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -280,8 +276,7 @@ public class EntityCreationService {
         "this is some text. a here is more text" and "here is more text". We only want to keep the latter.
          */
         return entityTextRanges.stream()
-                .filter(boundary -> entityTextRanges.stream()
-                        .noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary)))
+                .filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary)))
                 .toList();
 
     }
@@ -356,10 +351,10 @@ public class EntityCreationService {
 
         return tableNode.streamTableCells()
                 .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
-                                                                       tableCell,
-                                                                       type,
-                                                                       entityType,
-                                                                       tableNode));
+                        tableCell,
+                        type,
+                        entityType,
+                        tableNode));
     }
 
 
@@ -367,10 +362,10 @@ public class EntityCreationService {
 
         return tableNode.streamTableCells()
                 .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()),
-                                                                       tableCell,
-                                                                       type,
-                                                                       entityType,
-                                                                       tableNode));
+                        tableCell,
+                        type,
+                        entityType,
+                        tableNode));
     }
 
 
@@ -505,10 +500,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) {
 
-        return node.streamAllSubNodesOfType(NodeType.PARAGRAPH)
-                .map(semanticNode -> bySemanticNode(semanticNode, type, entityType))
-                .filter(Optional::isPresent)
-                .map(Optional::get);
+        return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get);
     }
 
 
@@ -598,18 +590,11 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        if (trimmedTextRange.length() == 0) {
-            return Optional.empty();
-        }
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
         if (node.getEntities().contains(entity)) {
-            Optional<TextEntity> optionalTextEntity = node.getEntities()
-                    .stream()
-                    .filter(e -> e.equals(entity) && e.type().equals(type))
-                    .peek(e -> e.addEngines(engines))
-                    .findAny();
+            Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
             if (optionalTextEntity.isEmpty()) {
-                return Optional.empty(); // Entity has been recategorized and should not be created at all.
+                return optionalTextEntity; // Entity has been recategorized and should not be created at all.
             }
             TextEntity existingEntity = optionalTextEntity.get();
             if (existingEntity.getTextRange().equals(textRange)) {
@@ -621,7 +606,7 @@ public class EntityCreationService {
             }
             return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
         }
-        addEntityToGraph(entity, node.getDocumentTree());
+        addEntityToGraph(entity, node);
         entity.addEngines(engines);
         insertToKieSession(entity);
         return Optional.of(entity);
@@ -650,8 +635,6 @@ public class EntityCreationService {
     }
 
 
-    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
-    @Deprecated(forRemoval = true)
     public TextEntity mergeEntitiesOfSameType(List<TextEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
 
         if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
@@ -664,44 +647,27 @@ public class EntityCreationService {
             return entitiesToMerge.get(0);
         }
 
-        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream()
-                                                                                       .map(TextEntity::getTextRange)
-                                                                                       .toList()), type, entityType, node);
-        mergedEntity.addEngines(entitiesToMerge.stream()
-                                        .flatMap(entityNode -> entityNode.getEngines()
-                                                .stream())
-                                        .collect(Collectors.toSet()));
-        entitiesToMerge.stream()
-                .map(TextEntity::getMatchedRuleList)
-                .flatMap(Collection::stream)
-                .forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
+        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node);
+        mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
+        entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
         entitiesToMerge.stream()
                 .map(TextEntity::getManualOverwrite)
                 .map(ManualChangeOverwrite::getManualChangeLog)
                 .flatMap(Collection::stream)
                 .forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange));
 
-        mergedEntity.setDictionaryEntry(entitiesToMerge.stream()
-                                                .anyMatch(TextEntity::isDictionaryEntry));
-        mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream()
-                                                       .anyMatch(TextEntity::isDossierDictionaryEntry));
-
-        entityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock());
+        mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDictionaryEntry));
+        mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDossierDictionaryEntry));
 
         addEntityToGraph(mergedEntity, node);
         insertToKieSession(mergedEntity);
-
-        entitiesToMerge.stream()
-                .filter(e -> !e.equals(mergedEntity))
-                .forEach(node.getEntities()::remove);
         return mergedEntity;
     }
 
 
     public Stream<TextEntity> copyEntities(List<TextEntity> entities, String type, EntityType entityType, SemanticNode node) {
 
-        return entities.stream()
-                .map(entity -> copyEntity(entity, type, entityType, node));
+        return entities.stream().map(entity -> copyEntity(entity, type, entityType, node));
     }
 
 
@@ -775,19 +741,25 @@ public class EntityCreationService {
     public void addEntityToGraph(TextEntity entity, SemanticNode node) {
 
         DocumentTree documentTree = node.getDocumentTree();
-        if (node.getEntities().contains(entity)) {
-            // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
-            node.getEntities()
-                    .stream()//
-                    .filter(e -> e.equals(entity))//
-                    .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
-                    .findAny()
-                    .ifPresent(e -> addDuplicateEntityToGraph(e, entity.getTextRange(), node));
-
-        } else {
-            addEntityToGraph(entity, documentTree);
+        try {
+            if (node.getEntities().contains(entity)) {
+                // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
+                node.getEntities().stream()//
+                        .filter(e -> e.equals(entity))//
+                        .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
+                        .findAny()//
+                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
+            } else {
+                entity.addIntersectingNode(documentTree.getRoot().getNode());
+                addEntityToGraph(entity, documentTree);
+            }
+        } catch (NoSuchElementException e) {
+            entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
+            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
+            entity.addIntersectingNode(documentTree.getRoot().getNode());
+            addToPages(entity);
+            addEntityToNodeEntitySets(entity);
         }
-
     }
 
 
@@ -798,11 +770,10 @@ public class EntityCreationService {
         SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
                 .stream()
                 .sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
-                .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges()
-                                                    .stream()
-                                                    .allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
-                                            intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
-                .findFirst().orElse(node.getDocumentTree().getRoot().getNode());
+                .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
+                        intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
+                .findFirst()
+                .orElse(node.getDocumentTree().getRoot().getNode());
 
         entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode);
 
@@ -813,8 +784,7 @@ public class EntityCreationService {
                 return;
             }
             additionalIntersectingNode.getEntities().add(entityToDuplicate);
-            additionalIntersectingNode.getPages(newTextRange)
-                    .forEach(page -> page.getEntities().add(entityToDuplicate));
+            additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate));
             entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
         });
     }
@@ -822,7 +792,12 @@ public class EntityCreationService {
 
     private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
 
-        documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity);
+        SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
+                .filter(node -> node.getTextBlock().containsTextRange(entity.getTextRange()))
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
+
+        containingNode.addThisToEntityIfIntersects(entity);
 
         TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
         entityEnrichmentService.enrichEntity(entity, textBlock);
@@ -831,4 +806,5 @@ public class EntityCreationService {
         addEntityToNodeEntitySets(entity);
     }
 
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -47,9 +47,7 @@ public class EntityFindingUtility {
     }
 
 
-    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
-                                                                          Map<String, List<TextEntity>> entitiesWithSameValue,
-                                                                          double matchThreshold) {
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
 
         if (precursorEntity.getValue() == null) {
             return Optional.empty();
@@ -58,7 +56,7 @@ public class EntityFindingUtility {
         List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
 
         if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
+            log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
             return Optional.empty();
         }
 
@@ -68,22 +66,18 @@ public class EntityFindingUtility {
                 .min(Comparator.comparingDouble(ClosestEntity::getDistance));
 
         if (optionalClosestEntity.isEmpty()) {
-            log.info("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
+            log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
             return Optional.empty();
         }
 
         ClosestEntity closestEntity = optionalClosestEntity.get();
         if (closestEntity.getDistance() > matchThreshold) {
-            log.info("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
-                     precursorEntity.getValue(),
-                     precursorEntity.getEntityPosition()
-                             .get(0).pageNumber(),
-                     precursorEntity.getEntityPosition()
-                             .stream()
-                             .map(RectangleWithPage::rectangle2D)
-                             .toList(),
-                     closestEntity.getDistance(),
-                     matchThreshold);
+            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                    precursorEntity.getValue(),
+                    precursorEntity.getEntityPosition().get(0).pageNumber(),
+                    precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    closestEntity.getDistance(),
+                    matchThreshold);
             return Optional.empty();
         }
 
@@ -99,14 +93,8 @@ public class EntityFindingUtility {
 
     private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
 
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage()
-                .stream()
-                .map(PositionOnPage::getPage)
-                .map(Page::getNumber)
-                .collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream()
-                .map(RectangleWithPage::pageNumber)
-                .collect(Collectors.toSet());
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
         return entityPageNumbers.containsAll(originalPageNumbers);
     }
 
@@ -117,16 +105,15 @@ public class EntityFindingUtility {
             return Double.MAX_VALUE;
         }
         return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .average()
                 .orElse(Double.MAX_VALUE);
     }
 
 
     private static long countRectangles(TextEntity entity) {
 
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
     }
 
 
@@ -174,8 +161,7 @@ public class EntityFindingUtility {
                     pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
                     node.getPages()));
         }
-
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues.stream().map(String::trim).collect(Collectors.toSet()), true);
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -9,6 +9,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@@ -22,21 +23,29 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
 import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-@RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityFromPrecursorCreationService {
 
     static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
     EntityFindingUtility entityFindingUtility;
+    EntityCreationService entityCreationService;
     DictionaryService dictionaryService;
 
 
+    @Autowired
+    public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
+
+        this.entityFindingUtility = entityFindingUtility;
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
     public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
 
         Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -57,18 +57,23 @@ public class SectionFinderService {
                 if (entry.getContainingNodeId().isEmpty()) {
                     continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
                 }
-                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
+                sectionsToReanalyse.add(entry.getContainingNodeId()
+                                                .get(0));
             }
         }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
+                                                                          .stream()
+                                                                          .map(DictionaryIncrementValue::getValue)
+                                                                          .toList(), true);
 
-        document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
-            }
-        });
+        document.streamChildren()
+                .forEach(mainNode -> {
+                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                        sectionsToReanalyse.add(mainNode.getTreeId()
+                                                        .get(0));
+                    }
+                });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
         relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
@@ -87,8 +92,11 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId().get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody()
+                                                       .stream()
+                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId()
+                        .get(0))
                 .toList();
     }
 
@@ -99,8 +107,7 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions()
-                .getEntriesToAdd()
+        return analyzeRequest.getManualRedactions().getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -132,11 +139,22 @@ public class SectionFinderService {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                                     .stream()
+                                     .map(ManualResizeRedaction::getAnnotationId),
+                             Stream.concat(manualRedactions.getLegalBasisChanges()
+                                                   .stream()
+                                                   .map(ManualLegalBasisChange::getAnnotationId),
+                                           Stream.concat(manualRedactions.getRecategorizations()
+                                                                 .stream()
+                                                                 .map(ManualRecategorization::getAnnotationId),
+                                                         Stream.concat(manualRedactions.getIdsToRemove()
+                                                                               .stream()
+                                                                               .map(IdRemoval::getAnnotationId),
+                                                                       manualRedactions.getForceRedactions()
+                                                                               .stream()
+                                                                               .map(ManualForceRedaction::getAnnotationId)))))
+                .collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -50,9 +50,7 @@ public class ComponentDroolsExecutionService {
                 .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
                 .map(entry -> Entity.fromEntityLogEntry(entry, document))
                 .forEach(kieSession::insert);
-        fileAttributes.stream()
-                .filter(f -> f.getValue() != null)
-                .forEach(kieSession::insert);
+        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
             kieSession.fireAllRules();
@@ -60,8 +58,7 @@ public class ComponentDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
@@ -74,9 +71,7 @@ public class ComponentDroolsExecutionService {
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        List<Component> components = getComponents(kieSession).stream()
-                .sorted(ComponentComparator.first())
-                .toList();
+        List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList();
         kieSession.dispose();
         return components;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -12,24 +15,34 @@ import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
+import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DroolsSyntaxValidationService {
-    
+
     private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
+    private final DeprecatedElementsFinder deprecatedElementsFinder;
+
 
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
@@ -43,15 +56,74 @@ public class DroolsSyntaxValidationService {
         }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
+        droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings());
         return droolsCompilerSyntaxValidation;
     }
 
+    private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
+
+        if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
+            String imports = ruleFileBluePrint.getImports();
+            SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation();
+            List<SearchImplementation.MatchPosition> matches = classesSearchImplementation.getMatches(imports);
+            if (!matches.isEmpty()) {
+                String sb = "Following imports are deprecated: \n" + matches.stream()
+                        .map(m -> imports.substring(m.startIndex(), m.endIndex()))
+                        .collect(Collectors.joining("\n"));
+                return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb)
+                                            .build();
+            }
+        }
+        return null;
+    }
+
+    private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
+
+        List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
+
+        // checks the rules for deprecateMethods
+        if (!deprecatedElementsFinder.getDeprecatedMethods().isEmpty()) {
+            SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation();
+            Map<String, String> deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap();
+
+            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
+                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
+                    for (BasicRule basicRule : ruleUnit.rules()) {
+                        List<SearchImplementation.MatchPosition> matches = methodsSearchImplementation.getMatches(basicRule.getCode());
+                        if (!matches.isEmpty()) {
+                            String warningMessage = matches.stream()
+                                    .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex()))
+                                    .distinct()
+                                    .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
+                                    .collect(Collectors.joining("\n"));
+                            warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage)
+                                                        .build());
+                        }
+                    }
+                }
+            }
+
+        }
+
+        return warningMessages.stream()
+                .sorted(Comparator.comparingInt(DroolsSyntaxDeprecatedWarnings::getLine))
+                .collect(Collectors.toList());
+    }
+
 
     private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
 
+        // find deprecated elements in the ruleFileBluePrint
+        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
+        if (warningMessageForImports != null) {
+            customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports);
+        }
+        customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
+
         RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
             case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
@@ -60,35 +132,36 @@ public class DroolsSyntaxValidationService {
         if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getImportLine())
-                            .column(0)
-                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                            .build());
+                                 .line(ruleFileBluePrint.getImportLine())
+                                 .column(0)
+                                 .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
+                                 .build());
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0)
-                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                            .build());
+                                 .line(ruleFileBluePrint.getGlobalsLine())
+                                 .column(0)
+                                 .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                                 .build());
         }
-        baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
-            if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicQuery.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                .build());
-            }
-        });
+        baseRuleFileBluePrint.getQueries()
+                .forEach(basicQuery -> {
+                    if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
+                        customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                                .add(DroolsSyntaxErrorMessage.builder()
+                                             .line(basicQuery.getLine())
+                                             .column(0)
+                                             .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
+                                             .build());
+                    }
+                });
         if (ruleFileType.equals(RuleFileType.ENTITY)) {
             String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
             if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
-                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                .build());
+                customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                        .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
+                                     .build());
             }
         }
         return customSyntaxValidation;
@@ -97,7 +170,8 @@ public class DroolsSyntaxValidationService {
 
     private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+        return ruleFileBluePrint.streamAllRules()
+                .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
     }
 
 
@@ -109,13 +183,16 @@ public class DroolsSyntaxValidationService {
 
         Set<String> additionalImports = Sets.difference(imports, baseImports);
 
-        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
+        return additionalImports.stream()
+                .allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
     private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
+        return ruleFileBluePrint.getQueries()
+                .stream()
+                .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
     }
 
 
@@ -124,9 +201,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId,
-                rules.getRulesString(),
-                RuleFileType.valueOf(rules.getRuleFileType()));
+                                                                                           versionId,
+                                                                                           rules.getRulesString(),
+                                                                                           RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 
@@ -134,14 +211,18 @@ public class DroolsSyntaxValidationService {
     private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
 
         List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
+        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
+                .map(this::buildDroolsSyntaxErrorMessage)
+                .collect(Collectors.toList());
+        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages)
+                .build();
     }
 
 
     private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
 
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
+        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText())
+                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -122,7 +122,7 @@ public class EntityDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
                     .get();
         } catch (ExecutionException e) {
             kieSession.dispose();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -1,327 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-import java.util.zip.GZIPInputStream;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Import;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.tenantcommons.TenantsClient;
-
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
-@Disabled
-/*
- * This test is meant to be used directly with a download from blob storage (e.g. minio). You need to define the dossier template you want to use by supplying an absolute path.
- * The dossier template will then be parsed for dictionaries, colors, entities, and rules. This is defined for the all tests once.
- * Inside a test you supply a path to your minio download folder. The files should still be zipped in this folder.
- * The files will then be checked for completeness and uploaded to the FileSystemBackedStorageService.
- * This way you can recreate what is happening on the stack almost exactly.
- */ public class AnalysisEnd2EndTest {
-
-    Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/business-logic/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_GFL_v1"); // Add your dossier-template here
-    ObjectMapper mapper = ObjectMapperFactory.create();
-    final String TENANT_ID = "tenant";
-
-    @Autowired
-    StorageService storageService;
-
-    @Autowired
-    protected AnalyzeService analyzeService;
-
-    @MockBean
-    DictionaryService dictionaryService;
-
-    @MockBean
-    RabbitTemplate rabbitTemplate;
-
-    TestDossierTemplate testDossierTemplate;
-    @MockBean
-    protected LegalBasisClient legalBasisClient;
-
-    @MockBean
-    private TenantsClient tenantsClient;
-
-    @MockBean
-    protected RulesClient rulesClient;
-
-    @MockBean
-    protected DictionaryClient dictionaryClient;
-
-
-    @Test
-    @SneakyThrows
-    public void runAnalysisEnd2End() {
-
-        String folder = "files/end2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple fileIds.
-
-        Path absoluteFolderPath;
-        if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
-            ClassPathResource classPathResource = new ClassPathResource(folder);
-            absoluteFolderPath = classPathResource.getFile().toPath();
-        } else {
-            absoluteFolderPath = Path.of(folder);
-        }
-
-        log.info("Starting end2end analyses for all distinct filenames in folder: {}", folder);
-        List<AnalyzeRequest> analyzeRequests = prepareStorageForFolder(absoluteFolderPath);
-        log.info("Found {} distinct fileIds", analyzeRequests.size());
-        for (int i = 0; i < analyzeRequests.size(); i++) {
-            AnalyzeRequest analyzeRequest = analyzeRequests.get(i);
-            log.info("{}/{}: Starting analysis for file {}", i + 1, analyzeRequests.size(), analyzeRequest.getFileId());
-            analyzeService.analyze(analyzeRequest);
-        }
-    }
-
-
-    @BeforeEach
-    public void setup() {
-
-        testDossierTemplate = new TestDossierTemplate(dossierTemplateToUse);
-        when(dictionaryService.updateDictionary(any(), any())).thenReturn(new DictionaryVersion(0, 0));
-        when(dictionaryService.getDeepCopyDictionary(any(), any())).thenReturn(testDossierTemplate.testDictionary);
-        when(dictionaryService.getDictionaryIncrements(any(), any(), any())).thenReturn(new DictionaryIncrement(Collections.emptySet(), new DictionaryVersion(0, 0)));
-        when(dictionaryService.isHint(any(String.class), any())).thenAnswer(invocation -> {
-            String type = invocation.getArgument(0);
-            return testDossierTemplate.testDictionary.getType(type).isHint();
-        });
-        when(dictionaryService.getColor(any(String.class), any())).thenAnswer(invocation -> {
-            String type = invocation.getArgument(0);
-            return testDossierTemplate.testDictionary.getType(type).getColor();
-        });
-        when(dictionaryService.getNotRedactedColor(any())).thenReturn(new float[]{0.2f, 0.2f, 0.2f});
-
-        when(rulesClient.getVersion(testDossierTemplate.id, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
-        when(rulesClient.getRules(testDossierTemplate.id, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(testDossierTemplate.rules));
-        when(rulesClient.getVersion(testDossierTemplate.id, RuleFileType.COMPONENT)).thenReturn(testDossierTemplate.componentRules != null ? System.currentTimeMillis() : -1);
-        when(rulesClient.getRules(testDossierTemplate.id, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(testDossierTemplate.componentRules));
-    }
-
-
-    @SneakyThrows
-    private List<AnalyzeRequest> prepareStorageForFolder(Path folder) {
-
-        return Files.list(folder)
-                .map(this::parseFileId)
-                .distinct()
-                .map(fileId -> prepareStorageForFile(fileId, folder))
-                .toList();
-    }
-
-
-    private String parseFileId(Path path) {
-
-        return path.getFileName().toString().split("\\.")[0];
-    }
-
-
-    @SneakyThrows
-    private AnalyzeRequest prepareStorageForFile(String fileId, Path folder) {
-
-        AnalyzeRequest request = new AnalyzeRequest();
-        request.setDossierId(UUID.randomUUID().toString());
-        request.setFileId(UUID.randomUUID().toString());
-        request.setDossierTemplateId(testDossierTemplate.id);
-        request.setManualRedactions(new ManualRedactions());
-        request.setAnalysisNumber(-1);
-
-        Set<FileType> endingsToUpload = Set.of("ORIGIN",
-                                               "DOCUMENT_PAGES",
-                                               "DOCUMENT_POSITION",
-                                               "DOCUMENT_STRUCTURE",
-                                               "DOCUMENT_TEXT",
-                                               "IMAGE_INFO",
-                                               "NER_ENTITIES",
-                                               "TABLES",
-                                               "IMPORTED_REDACTIONS")
-                .stream()
-                .map(FileType::valueOf)
-                .collect(Collectors.toSet());
-
-        Set<FileType> uploadedFileTypes = Files.walk(folder)
-                .filter(path -> path.toFile().isFile())
-                .filter(path -> endingsToUpload.contains(parseFileTypeFromPath(path)))
-                .map(filePath -> uploadFile(filePath, request))
-                .collect(Collectors.toUnmodifiableSet());
-
-        Set<FileType> missingFileTypes = Sets.difference(endingsToUpload, uploadedFileTypes);
-        if (!missingFileTypes.isEmpty()) {
-            log.error("Folder {} is missing files of type {}",
-                      folder.toFile(),
-                      missingFileTypes.stream()
-                              .map(Enum::toString)
-                              .collect(Collectors.joining(", ")));
-            throw new NotFoundException("Not all required file types are present.");
-        }
-        return request;
-    }
-
-
-    private static FileType parseFileTypeFromPath(Path path) {
-
-        return FileType.valueOf(path.getFileName().toString().split("\\.")[1]);
-    }
-
-
-    @SneakyThrows
-    private FileType uploadFile(Path path, AnalyzeRequest request) {
-
-        FileType fileType = parseFileTypeFromPath(path);
-        try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis);) {
-            storageService.storeObject(TENANT_ID, RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType), in);
-
-        }
-        return fileType;
-    }
-
-
-    private class TestDossierTemplate {
-
-        String id;
-        Dictionary testDictionary;
-        AtomicInteger dictEntryIdCounter = new AtomicInteger(0);
-        String rules;
-        String componentRules;
-
-
-        @SneakyThrows
-        TestDossierTemplate(Path dossierTemplateToUse) {
-
-            Map<String, Object> dossierTemplate = mapper.readValue(dossierTemplateToUse.resolve("dossierTemplate.json").toFile(), HashMap.class);
-            this.id = (String) dossierTemplate.get("dossierTemplateId");
-            List<DictionaryModel> dictionaries = Files.walk(dossierTemplateToUse)
-                    .filter(path -> path.getFileName().toString().equals("dossierType.json"))
-                    .map(this::loadDictionaryModel)
-                    .toList();
-            File ruleFile = dossierTemplateToUse.resolve("rules.drl").toFile();
-            rules = new String(Files.readAllBytes(ruleFile.toPath()));
-
-            File componentRuleFile = dossierTemplateToUse.resolve("componentRules.drl").toFile();
-            if (componentRuleFile.exists()) {
-                componentRules = new String(Files.readAllBytes(componentRuleFile.toPath()));
-            }
-
-            testDictionary = new Dictionary(dictionaries, new DictionaryVersion(0, 0));
-        }
-
-
-        @SneakyThrows
-        private DictionaryModel loadDictionaryModel(Path path) {
-
-            Map<String, Object> model = mapper.readValue(path.toFile(), HashMap.class);
-            Set<DictionaryEntryModel> entries = new HashSet<>();
-            Set<DictionaryEntryModel> falsePositives = new HashSet<>();
-            Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
-
-            String type = (String) model.get("type");
-            Integer rank = (Integer) model.get("rank");
-            float[] color = hexToFloatArr((String) model.get("hexColor"));
-            Boolean caseInsensitive = (Boolean) model.get("caseInsensitive");
-            Boolean hint = (Boolean) model.get("hint");
-            Boolean hasDictionary = (Boolean) model.get("hasDictionary");
-
-            boolean isDossierDictionary;
-            if (model.containsKey("dossierDictionaryOnly")) {
-                isDossierDictionary = true;
-            } else {
-                isDossierDictionary = ((String) model.get("id")).split(":").length == 3;
-            }
-
-            if (hasDictionary) {
-                try (var in = new FileInputStream(path.getParent().resolve("entries.txt").toFile())) {
-                    entries.addAll(parseDictionaryEntryModelFromFile(new String(in.readAllBytes()), dictEntryIdCounter, (String) model.get("typeId")));
-                }
-                try (var in = new FileInputStream(path.getParent().resolve("falsePositives.txt").toFile())) {
-                    falsePositives.addAll(parseDictionaryEntryModelFromFile(new String(in.readAllBytes()), dictEntryIdCounter, (String) model.get("typeId")));
-                }
-                try (var in = new FileInputStream(path.getParent().resolve("falseRecommendations.txt").toFile())) {
-                    falseRecommendations.addAll(parseDictionaryEntryModelFromFile(new String(in.readAllBytes()), dictEntryIdCounter, (String) model.get("typeId")));
-                }
-            }
-
-            return new DictionaryModel(type, rank, color, caseInsensitive, hint, entries, falsePositives, falseRecommendations, isDossierDictionary);
-        }
-
-
-        private Set<DictionaryEntryModel> parseDictionaryEntryModelFromFile(String s, AtomicInteger dictEntryIdCounter, String typeId) {
-
-            String[] values = s.split("\n");
-            return Arrays.stream(values)
-                    .map(value -> new DictionaryEntryModel(dictEntryIdCounter.getAndIncrement(), value, 0L, false, typeId))
-                    .collect(Collectors.toUnmodifiableSet());
-        }
-
-
-        private float[] hexToFloatArr(String hexColor) {
-
-            // Remove # symbol if present
-            String cleanHexColor = hexColor.replace("#", "");
-
-            // Parse hex string into RGB components
-            int r = Integer.parseInt(cleanHexColor.substring(0, 2), 16);
-            int g = Integer.parseInt(cleanHexColor.substring(2, 4), 16);
-            int b = Integer.parseInt(cleanHexColor.substring(4, 6), 16);
-
-            // Normalize RGB values to floats between 0 and 1
-            float[] rgbFloat = new float[3];
-            rgbFloat[0] = r / 255.0f;
-            rgbFloat[1] = g / 255.0f;
-            rgbFloat[2] = b / 255.0f;
-
-            return rgbFloat;
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -177,9 +177,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
                                                                                               document,
                                                                                               TEST_DOSSIER_TEMPLATE_ID,
                                                                                               manualRedactions,
-                                                                                              TEST_FILE_ID,
-                                                                                              Collections.emptySet(),
-                                                                                              false);
+                                                                                              TEST_FILE_ID);
 
         redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                       .hexColor("#ffe187")
-                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
@@ -122,12 +122,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
     }
 
-
     @Test
     public void acceptanceTests() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -136,10 +133,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished analysis");
         EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
-                .orElseThrow();
-        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
-                .orElseThrow();
+        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
@@ -151,10 +146,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
-                .orElseThrow();
-        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
-                .orElseThrow();
+        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.APPLIED, asyaLyon2.getState());
 
@@ -175,25 +168,19 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
                 .filter(entry -> entry.getValue().equals(value))
-                .filter(entry -> entry.getContainingNodeId()
-                        .get(0).equals(sectionNumber.get(0)));
+                .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
     }
 
 
     private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
 
-        return redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value));
+        return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
     }
 
 
     @Test
     public void noEndlessLoopsTest() {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -214,15 +201,13 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         assertEquals(EntryState.IGNORED,
-                     findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY))
-                             .findFirst()
-                             .get().getState());
+                findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState());
     }
 
 
     private static IdRemoval buildIdRemoval(String id) {
 
-        return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
+        return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -218,12 +218,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = ManualRedactions.builder()
                 .resizeRedactions(Set.of(ManualResizeRedaction.builder()
                                                  .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
-                                                 .fileId(TEST_FILE_ID)
                                                  .value("Dr. Alan")
                                                  .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
                                                  .requestDate(OffsetDateTime.now())
                                                  .updateDictionary(false)
-                                                 .user("user")
                                                  .build()))
                 .build();
 
@@ -585,8 +583,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void redactionTest() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         String fileName = "files/new/crafted document.pdf";
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
@@ -660,7 +656,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                              .fileId("fileId")
                                                              .type("signature")
                                                              .requestDate(OffsetDateTime.now())
-                                                             .user("user")
                                                              .build()));
 
         request.setManualRedactions(manualRedactions);
@@ -715,7 +710,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                              .annotationId("308dab9015bfafd911568cffe0a7f7de")
                              .fileId(TEST_FILE_ID)
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                             .user("user")
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
                              .build());
 
@@ -724,7 +718,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                              .annotationId("0b56ea1a87c83f351df177315af94f0d")
                              .fileId(TEST_FILE_ID)
                              .legalBasis("Something")
-                             .user("user")
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
                              .build());
@@ -733,7 +726,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .add(IdRemoval.builder()
                              .annotationId("0b56ea1a87c83f351df177315af94f0d")
                              .fileId(TEST_FILE_ID)
-                             .user("user")
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
                              .build());
@@ -953,7 +945,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .textBefore("")
                 .updateDictionary(false)
                 .textAfter("")
-                .user("user")
                 .build();
         manualRedactions.getResizeRedactions().add(manualResizeRedaction);
 
@@ -1041,14 +1032,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                                                        .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                                                        .fileId("fileId")
-                                                       .user("user")
                                                        .processedDate(OffsetDateTime.now())
                                                        .requestDate(OffsetDateTime.now())
                                                        .build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                                                            .annotationId("675eba69b0c2917de55462c817adaa05")
                                                            .fileId("fileId")
-                                                           .user("user")
                                                            .legalBasis("Something")
                                                            .requestDate(OffsetDateTime.now())
                                                            .processedDate(OffsetDateTime.now())
@@ -1238,8 +1227,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void signaturesAreRedactionAfterReanalyse() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
@@ -1261,7 +1248,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
                                                                               .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                                                                              .user("user")
                                                                               .build()))
                                             .build());
 
@@ -1281,8 +1267,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void entityIsAppliedAfterRecateorize() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
@@ -1304,14 +1288,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
                                                                               .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                                                                              .user("user")
                                                                               .build()))
                                             .recategorizations(Set.of(ManualRecategorization.builder()
                                                                               .annotationId("3029651d0842a625f2d23f8375c23600")
                                                                               .type("CBI_author")
+                                                                              .legalBasis("")
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
-                                                                              .user("user")
                                                                               .build()))
                                             .build());
 
@@ -1455,8 +1438,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @SneakyThrows
     public void testResizeWithUpdateDictionaryTrue() {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         String pdfFile = "files/new/crafted document.pdf";
 
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -1474,8 +1455,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                             .resizeRedactions(Set.of(ManualResizeRedaction.builder()
                                                                              .updateDictionary(true)
                                                                              .annotationId(david.getId())
-                                                                             .fileId(TEST_FILE_ID)
-                                                                             .user("user")
                                                                              .requestDate(OffsetDateTime.now())
                                                                              .value("David Ksenia")
                                                                              .positions(List.of(Rectangle.builder()
@@ -1503,14 +1482,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
     private IdRemoval getIdRemoval(String id) {
 
-        return IdRemoval.builder()
-                .annotationId(id)
-                .removeFromAllDossiers(false)
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .removeFromDictionary(false)
-                .requestDate(OffsetDateTime.now())
-                .build();
+        return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java

@@ -5,12 +5,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.awt.Color;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.drools.io.ClassPathResource;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -18,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@@ -47,13 +49,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -62,14 +66,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithRemovedImportsAndChangedName() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
         rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -77,102 +83,32 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithAddedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
-    @Test
-    @SneakyThrows
-    void testRulesWithUnallowedAddedImports() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertFalse(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAllRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAcceptanceRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testManualRedactionRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testRulesV2() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
     @Test
     @SneakyThrows
     void testDocumineRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -181,7 +117,8 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
@@ -196,12 +133,13 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
-                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
+                                                       "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -212,14 +150,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRemoveQuery() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", "");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -228,13 +168,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -243,13 +185,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDroolsAsEntityDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -259,13 +203,15 @@ class DroolsSyntaxValidationServiceTest {
     @Disabled
     void attemptImportsFixToAllRuleFiles() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
 
-        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
-                "drools/documine_flora.drl",
-                "drools/manual_redaction_rules.drl",
-                "drools/acceptance_rules.drl",
-                "drools/rules_v2.drl");
+        List<String> ruleFiles = List.of("drools/rules.drl",
+                                         "drools/all_redact_manager_rules.drl",
+                                         "drools/documine_flora.drl",
+                                         "drools/manual_redaction_rules.drl",
+                                         "drools/acceptance_rules.drl",
+                                         "drools/rules_v2.drl");
 
         for (String ruleFile : ruleFiles) {
             var rulesFile = new ClassPathResource(ruleFile);
@@ -288,4 +234,109 @@ class DroolsSyntaxValidationServiceTest {
 
     }
 
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAllRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAcceptanceRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testManualRedactionRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesV2() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesWithDeprecatedFunctions() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert(");
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+        assertEquals(droolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().size(), 2);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                                                                                                        .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                        .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                        .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                        .hexColor("#ffe187")
-                                                                                                        .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
@@ -155,40 +155,29 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String testEntityValue1 = "Desiree";
         String testEntityValue2 = "Melanie";
         EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertEquals(2,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue1))
-                             .count());
-        assertEquals(2,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue2))
-                             .count());
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
-        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document)
-                .findFirst()
-                .get();
+        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
         String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getValue().equals(testEntityValue1))
                 .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
-                .get().getId();
+                .get()
+                .getId();
         ManualRedactions manualRedactions = new ManualRedactions();
-        manualRedactions.getResizeRedactions()
-                .add(ManualResizeRedaction.builder()
-                             .annotationId(idToResize)
-                             .fileId(TEST_FILE_ID)
-                             .value(expandedEntityKeyword)
-                             .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
-                                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
-                             .addToAllDossiers(false)
-                             .updateDictionary(false)
-                             .requestDate(OffsetDateTime.now())
-                             .build());
+        manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
+                        .annotationId(idToResize)
+                        .value(expandedEntityKeyword)
+                        .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
+                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
+                        .addToAllDossiers(false)
+                        .updateDictionary(false)
+                        .requestDate(OffsetDateTime.now())
+                .build());
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
@@ -199,32 +188,21 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
-        EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getValue().equals(expandedEntityKeyword))
-                .findFirst()
-                .get();
+        EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
         assertEquals(idToResize, resizedEntry.getId());
+        assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
         assertEquals(1,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue1))
-                             .count());
-        assertEquals(1,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED))
-                             .count());
+                redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count());
     }
 
 
     private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
         return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
-                                                                                                       (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
-                                                                                                       (float) rectangle2D.getWidth(),
-                                                                                                       -(float) rectangle2D.getHeight(),
-                                                                                                       pageNumber);
+                (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 
@@ -241,10 +219,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
-                                                           .annotationId("675eba69b0c2917de55462c817adaa05")
-                                                           .fileId("fileId")
-                                                           .legalBasis("Something")
-                                                           .build()));
+                .annotationId("675eba69b0c2917de55462c817adaa05")
+                .fileId("fileId")
+                .legalBasis("Something")
+                .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
@@ -254,7 +232,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactionEntry.setValue("O'Loughlin C.K.");
         manualRedactionEntry.setReason("Manual Redaction");
         manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                                                  Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
+                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
 
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
@@ -264,11 +242,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
-                                                              .annotationId("675eba69b0c2917de55462c817adaa05")
-                                                              .fileId("fileId")
-                                                              .legalBasis("Manual Legal Basis Change")
-                                                              .requestDate(OffsetDateTime.now())
-                                                              .build())));
+                .annotationId("675eba69b0c2917de55462c817adaa05")
+                .fileId("fileId")
+                .legalBasis("Manual Legal Basis Change")
+                .requestDate(OffsetDateTime.now())
+                .build())));
 
         analyzeService.reanalyze(request);
 
@@ -317,10 +295,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long end = System.currentTimeMillis();
-        var optionalEntry = redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
-                .findAny();
+        var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
         assertTrue(optionalEntry.isPresent());
         assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
         System.out.println("duration: " + (end - start));
@@ -370,9 +345,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         assertFalse(redactionLog2.getEntityLogEntry()
-                            .stream()
-                            .filter(entry -> entry.getType().equals("published_information"))
-                            .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
+                .stream()
+                .filter(entry -> entry.getType().equals("published_information"))
+                .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
 
         var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry()
                 .stream()
@@ -406,15 +381,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String annotationId = "testAnnotationId";
 
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
-                                                        .annotationId(annotationId)
-                                                        .requestDate(OffsetDateTime.now())
-                                                        .type("manual")
-                                                        .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
-                                                        .positions(List.of(//
-                                                                           new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
-                                                                           new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
-                                                                           new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
-                                                        .build()));
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .type("manual")
+                .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
+                .positions(List.of(//
+                        new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
+                        new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
+                        new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
+                .build()));
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -32,33 +32,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualResizeRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
         Set<TextEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document)
                 .collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
-        TextEntity biggerEntity = biggerEntities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+        TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(initialId)
-                .fileId(TEST_FILE_ID)
-                .user("user")
                 .value(biggerEntity.getValue())
-                .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage()
-                                                          .get(0)))
+                .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
                 .requestDate(OffsetDateTime.now())
                 .updateDictionary(false)
                 .build();
@@ -70,13 +55,8 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
         assertEquals(biggerEntity.getPages(), entity.getPages());
         assertEquals(biggerEntity.getValue(), entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
-        assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage()
-                                            .get(0).getRectanglePerLine(),
-                                    entity.getPositionsOnPagePerPage()
-                                            .get(0).getRectanglePerLine());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
+        assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine());
         assertTrue(entity.resized());
     }
 
@@ -85,25 +65,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualForceRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -111,12 +78,8 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertFalse(entity.getIntersectingNodes().isEmpty());
         assertEquals(1, entity.getPages().size());
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals("Something",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertFalse(entity.removed());
         assertTrue(entity.hasManualChanges());
         assertTrue(entity.applied());
@@ -127,26 +90,17 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualIDRemovalTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).user("user").build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(idRemoval));
 
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertTrue(entity.ignored());
     }
 
@@ -155,25 +109,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualIDRemovalButAlsoForceRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -181,9 +123,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertFalse(entity.getIntersectingNodes().isEmpty());
         assertEquals(1, entity.getPages().size());
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertFalse(entity.removed());
         assertFalse(entity.ignored());
     }
@@ -191,9 +131,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
     private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
 
-        if (rects1.stream()
-                .allMatch(rect1 -> rects2.stream()
-                        .anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
+        if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
             return;
         }
         // use this for nice formatting of error message
@@ -205,18 +143,15 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
         double tolerance = 1e-1;
         return Math.abs(r1.getX() - r2.getX()) < tolerance &&//
-               Math.abs(r1.getY() - r2.getY()) < tolerance &&//
-               Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
-               Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
+                Math.abs(r1.getY() - r2.getY()) < tolerance &&//
+                Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
+                Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
     }
 
 
     private static List<Rectangle> toAnnotationRectangles(PositionOnPage positionsOnPage) {
 
-        return positionsOnPage.getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber()))
-                .toList();
+        return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java

@@ -43,9 +43,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         OffsetDateTime start = OffsetDateTime.now();
         String reason = "whatever";
         Document document = buildGraphNoImages("files/new/crafted document.pdf");
-        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document)
-                .peek(e -> e.apply("T.0.0", reason))
-                .toList();
+        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList();
         assertFalse(entities.isEmpty());
         TextEntity entity = entities.get(0);
         assertTrue(entity.active());
@@ -54,11 +52,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.resized());
         assertFalse(entity.ignored());
         assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
-        String annotationId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).user("user").annotationId(annotationId).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -68,7 +65,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         ManualForceRedaction forceRedaction = ManualForceRedaction.builder()
                 .requestDate(start.plusSeconds(1))
                 .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
                 .legalBasis("coolio")
                 .build();
@@ -77,12 +73,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.ignored());
         assertFalse(entity.removed());
         assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
-        assertEquals("coolio",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
 
         // remove again
-        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
+        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal2);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -92,7 +86,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         ManualForceRedaction forceRedaction2 = ManualForceRedaction.builder()
                 .requestDate(start.plusSeconds(2))
                 .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
                 .legalBasis("coolio")
                 .build();
@@ -100,7 +93,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
         assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override",
-                     entity.buildReasonWithManualChangeDescriptions());
+                entity.buildReasonWithManualChangeDescriptions());
 
         String legalBasis = "Yeah";
         String section = "Some random section!";
@@ -110,7 +103,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .annotationId(annotationId)
                 .requestDate(start.plusSeconds(4))
                 .section(section)
-                .fileId(TEST_FILE_ID)
                 .user("peter")
                 .value(value)
                 .build();
@@ -118,32 +110,21 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
         assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed",
-                     entity.buildReasonWithManualChangeDescriptions());
-        assertEquals(value,
-                     entity.getManualOverwrite().getValue()
-                             .orElse(entity.getValue()));
-        assertEquals(legalBasis,
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
-        assertEquals(section,
-                     entity.getManualOverwrite().getSection()
-                             .orElse(entity.getDeepestFullyContainingNode().toString()));
+                entity.buildReasonWithManualChangeDescriptions());
+        assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue()));
+        assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
 
         ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
                 .type("type")
                 .requestDate(start.plusSeconds(5))
-                .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
+                .legalBasis("")
                 .build();
         entity.getManualOverwrite().addChange(imageRecategorizationRequest);
-        assertTrue(entity.getManualOverwrite().getRecategorized()
-                           .isPresent());
-        assertTrue(entity.getManualOverwrite().getRecategorized()
-                           .get());
-        assertEquals("type",
-                     entity.getManualOverwrite().getType()
-                             .orElse(entity.type()));
+        assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
+        assertTrue(entity.getManualOverwrite().getRecategorized().get());
+        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
     }
 
 
@@ -153,9 +134,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         OffsetDateTime start = OffsetDateTime.now();
         String reason = "whatever";
         Document document = buildGraphNoImages("files/new/crafted document.pdf");
-        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document)
-                .peek(e -> e.apply("T.0.0", reason))
-                .toList();
+        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList();
         assertFalse(entities.isEmpty());
         TextEntity entity = entities.get(0);
         assertTrue(entity.active());
@@ -164,11 +143,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.resized());
         assertFalse(entity.ignored());
         assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
-        String annotationId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -179,7 +157,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .requestDate(start.plusSeconds(1))
                 .fileId(TEST_FILE_ID)
                 .annotationId(annotationId)
-                .user("user")
                 .legalBasis("coolio")
                 .build();
         entity.getManualOverwrite().addChange(forceRedaction);
@@ -187,9 +164,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.ignored());
         assertFalse(entity.removed());
         assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
-        assertEquals("coolio",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java

@@ -84,7 +84,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
     public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
 
         DocumentAndEntity context = createFoundManualRedaction();
-        IdRemoval removal = IdRemoval.builder().annotationId("123").user("user").fileId(TEST_FILE_ID).requestDate(OffsetDateTime.now()).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
         context.entity().getManualOverwrite().addChange(removal);
         assertTrue(context.entity().removed());
     }
@@ -95,7 +95,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
     public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
 
         DocumentAndEntity context = createNotFoundManualRedaction();
-        IdRemoval removal = IdRemoval.builder().fileId(TEST_FILE_ID).user("user").annotationId("123").requestDate(OffsetDateTime.now()).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
         context.entity().getManualOverwrite().addChange(removal);
         assertTrue(context.entity().removed());
     }
@@ -108,11 +108,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         String value = "To: Syngenta Ltd. Jealott’s Hill";
         String type = DICTIONARY_AUTHOR;
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
-                .annotationId("123")
                 .type(type)
                 .value(value)
-                .user("user")
-                .fileId(TEST_FILE_ID)
                 .reason("reason")
                 .legalBasis("n-a")
                 .section("n-a")
@@ -125,20 +122,17 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
 
         assertTrue(document.getEntities().isEmpty());
 
-        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
-                                                                                                                                                .entriesToAdd(Set.of(
-                                                                                                                                                        manualRedactionEntry))
-                                                                                                                                                .build(),
-                                                                                                                                        document,
-                                                                                                                                        TEST_DOSSIER_TEMPLATE_ID);
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertEquals(1, notFoundManualEntities.size());
         assertTrue(document.getEntities().isEmpty());
 
         List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
-                                                                                                  document,
-                                                                                                  notFoundManualEntities,
-                                                                                                  new DictionaryVersion(),
-                                                                                                  0L).getEntityLogEntry();
+                document,
+                notFoundManualEntities,
+                new DictionaryVersion(),
+                0L).getEntityLogEntry();
 
         assertEquals(1, redactionLogEntries.size());
         assertEquals(value, redactionLogEntries.get(0).getValue());
@@ -152,8 +146,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         Document document = buildGraph("files/new/VV-919901.pdf");
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
 
-        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
-                .toList();
+        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
         assertFalse(tempEntities.isEmpty());
         var tempEntity = tempEntities.get(0);
         List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
@@ -165,11 +158,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
 
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
                 .type("manual")
-                .annotationId("123")
                 .value(tempEntity.getValue())
                 .reason("reason")
-                .user("user")
-                .fileId(TEST_FILE_ID)
                 .legalBasis("n-a")
                 .section(tempEntity.getDeepestFullyContainingNode().toString())
                 .rectangle(false)
@@ -182,28 +172,21 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         tempEntity.removeFromGraph();
         assertTrue(document.getEntities().isEmpty());
 
-        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
-                                                                                                                                                .entriesToAdd(Set.of(
-                                                                                                                                                        manualRedactionEntry))
-                                                                                                                                                .build(),
-                                                                                                                                        document,
-                                                                                                                                        TEST_DOSSIER_TEMPLATE_ID);
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertTrue(notFoundManualEntities.isEmpty());
         assertEquals(1, document.getEntities().size());
-        return new DocumentAndEntity(document,
-                                     document.getEntities()
-                                             .stream()
-                                             .findFirst()
-                                             .get());
+        return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
     }
 
 
     public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
         return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
-                             (float) rectangle2D.getWidth(),
-                             -(float) rectangle2D.getHeight(),
-                             pageNumber);
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.rules;
 import java.util.Collection;
 
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -11,8 +10,6 @@ import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.kie.internal.io.ResourceFactory;
-import org.mockito.Mockito;
-import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Bean;
@@ -20,13 +17,11 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
-@ExtendWith(MockitoExtension.class)
 public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
     protected static final String RULES = "drools/rules.drl";
@@ -77,12 +72,10 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
     @BeforeEach
     public void createServices() {
 
-        Dictionary dict = Mockito.mock(Dictionary.class);
         kieSession = kieContainer.newKieSession();
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
         kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
         kieSession.setGlobal("entityCreationService", entityCreationService);
-        kieSession.setGlobal("dictionary", dict);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode() {
 
-        var node = new Section(List.of(0, 1), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(0, 1), null, null, null, null,null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -29,8 +29,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode2() {
 
-        var node = new Section(Collections.emptyList(), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(Collections.emptyList(), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -42,8 +42,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode3() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode4() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null,null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -8,7 +8,6 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -35,6 +34,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
@@ -84,7 +84,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
     @SpyBean
     RabbitTemplate rabbitTemplate;
 
-
     @BeforeEach
     public void stubClients() {
 
@@ -102,22 +101,21 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                       .hexColor("#ffe187")
-                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
         when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
     }
 
-
     @Test
     @SneakyThrows
     public void testManualSurroundingText() {
@@ -127,20 +125,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         ManualRedactions manualRedactions = new ManualRedactions();
 
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(355.53775f)
-                                                                                                .topLeftY(266.1895f)
-                                                                                                .width(29.32224f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "AOEL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL");
 
         var notFoundId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId,
-                                                                                 List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()),
-                                                                                 "Random");
+        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@@ -159,43 +147,30 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertFalse(unprocessedManualEntities.isEmpty());
         assertEquals(unprocessedManualEntities.size(), 2);
 
-        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream()
-                .filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId))
-                .findFirst();
+        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
         assertTrue(optionalUnprocessedManualEntity.isPresent());
 
         UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
         assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
         assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
         assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).x(), 355.53775f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).y(), 266.49002f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).w(), 29.322266f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).h(), 11.017679f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
 
-        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream()
-                .filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId))
-                .findFirst();
+        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
         assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
 
         UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
         assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
         assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
         assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getPageNumber(), 1);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[0], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[1], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[2], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[3], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
 
         analyzeService.reanalyze(request);
 
@@ -215,14 +190,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -234,37 +205,13 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         ManualRedactions manualRedactions = new ManualRedactions();
 
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         var cormsId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId,
-                                                                                 List.of(Rectangle.builder()
-                                                                                                 .topLeftX(129.86f)
-                                                                                                 .topLeftY(505.7295f)
-                                                                                                 .width(35.9904f)
-                                                                                                 .height(10.048125f)
-                                                                                                 .page(1)
-                                                                                                 .build()),
-                                                                                 "CoRMS");
+        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS");
 
         var a9Id = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id,
-                                                                                 List.of(Rectangle.builder()
-                                                                                                 .topLeftX(140.1096f)
-                                                                                                 .topLeftY(291.5095f)
-                                                                                                 .width(37.84512f)
-                                                                                                 .height(10.048125f)
-                                                                                                 .page(1)
-                                                                                                 .build()),
-                                                                                 "A9396G");
+        ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@@ -291,53 +238,35 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertFalse(unprocessedManualEntities.isEmpty());
         assertEquals(unprocessedManualEntities.size(), 3);
 
-        var resizedAoel = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId))
-                .findFirst();
+        var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst();
         assertTrue(resizedAoel.isPresent());
         assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
         assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
         assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight());
 
-        var cormsResized = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId))
-                .findFirst();
+        var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst();
         assertTrue(cormsResized.isPresent());
         assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
         assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
         assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).x(), positions2.get(0).getTopLeftX());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).y(), positions2.get(0).getTopLeftY());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).w(), positions2.get(0).getWidth());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).h(), positions2.get(0).getHeight());
+        assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX());
+        assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY());
+        assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth());
+        assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight());
 
-        var a9Resized = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id))
-                .findFirst();
+        var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst();
         assertTrue(a9Resized.isPresent());
         assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
         assertEquals(a9Resized.get().getTextBefore(), "data for S");
         assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).x(), positions3.get(0).getTopLeftX());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).y(), positions3.get(0).getTopLeftY());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).w(), positions3.get(0).getWidth());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).h(), positions3.get(0).getHeight());
+        assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX());
+        assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY());
+        assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth());
+        assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight());
     }
 
 
@@ -348,15 +277,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -380,14 +301,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -398,15 +315,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -430,14 +339,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -448,15 +353,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -480,32 +377,25 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
     private static ManualResizeRedaction prepareManualSizeRedaction(String id, List<Rectangle> positions, String value) {
 
-        return ManualResizeRedaction.builder()
-                .annotationId(id)
-                .fileId("fileId")
-                .user("user")
-                .positions(positions)
-                .updateDictionary(false)
-                .addToAllDossiers(false)
-                .value(value)
-                .requestDate(OffsetDateTime.now())
-                .build();
+        ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction();
+        manualResizeRedaction.setAnnotationId(id);
+        manualResizeRedaction.setPositions(positions);
+        manualResizeRedaction.setUpdateDictionary(false);
+        manualResizeRedaction.setAddToAllDossiers(false);
+        manualResizeRedaction.setValue(value);
+        return manualResizeRedaction;
     }
 
 
     private static ManualRedactionEntry prepareManualRedactionEntry(String id, List<Rectangle> positions, String value) {
 
-        return ManualRedactionEntry.builder()
-                .annotationId(id)
-                .fileId("fileId")
-                .user("user")
-                .type("CBI_author")
-                .value(value)
-                .reason("Manual Redaction")
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .positions(positions)
-                .build();
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(id);
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue(value);
+        manualRedactionEntry.setReason("Manual Redaction");
+        manualRedactionEntry.setPositions(positions);
+        return manualRedactionEntry;
     }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -16,6 +16,7 @@ public class LayoutParsingRequestProvider {
         var originFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN);
         var tablesFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES);
         var imagesFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO);
+        var sectionGridStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID);
         var structureFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE);
         var textBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT);
         var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
@@ -28,12 +29,14 @@ public class LayoutParsingRequestProvider {
                 .originFileStorageId(originFileStorageId)
                 .tablesFileStorageId(Optional.of(tablesFileStorageId))
                 .imagesFileStorageId(Optional.of(imagesFileStorageId))
+                .visualLayoutParsingFileId(Optional.empty())
                 .structureFileStorageId(structureFileStorageId)
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
                 .simplifiedTextStorageId(simplifiedTextStorageId)
                 .viewerDocumentStorageId(viewerDocumentStorageId)
+                .visualLayoutParsingFileId(Optional.empty())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml

@@ -25,8 +25,6 @@ redaction-service:
 application:
   type: "RedactManager"
 
-logging.type: "CONSOLE"
-
 storage:
   backend: 's3'
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -153,32 +155,6 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
 
-rule "CBI.7.2: Do not redact PII if published information found in Section without tables"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("published_information"),
-                          hasEntitiesOfType("PII"))
-    then
-        $section.getEntitiesOfType("PII")
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.2",
-                        "Published Information found in section",
-                        $section.getEntitiesOfType("published_information")
-                );
-            });
-    end
-
-rule "CBI.7.3: Do not redact PII if published information found in same table row"
-    when
-        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("PII"))
-        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
-        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $pii: TextEntity(type() == "PII", active()) from $tableCell.getEntities()
-    then
-        $pii.skipWithReferences("CBI.7.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
-    end
-
 
 // Rule unit: CBI.9
 rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
@@ -207,19 +183,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
             .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.10
 rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@@ -248,32 +211,6 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st
             .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.10.2: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "CBI.10.3: Redact all cells with Header Author as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.11
 rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
@@ -287,19 +224,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -307,12 +232,12 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -320,19 +245,7 @@ rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -357,19 +270,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 
 
 // Rule unit: CBI.20
-rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
+rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -377,12 +278,12 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
+             laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
+rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -390,117 +291,56 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\""
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.3", "PERFORMING LABORATORY was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.0: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "PII.0.2: Redact all PII (vertebrate study)"
+rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.3: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.3", "Personal Information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: PII.1
-rule "PII.1.0: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
+rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
+rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.5: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.5", "Found by Email Regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.2
-rule "PII.2.0: Redact Phone and Fax by RegEx"
+rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
@@ -513,28 +353,10 @@ rule "PII.2.0: Redact Phone and Fax by RegEx"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
+rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("Contact") ||
@@ -548,42 +370,35 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.3
-rule "PII.3.0: Redact telephone numbers by RegEx"
-    when
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
+rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
+rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.4
-rule "PII.4.0: Redact line after contact information keywords"
+rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -609,62 +424,7 @@ rule "PII.4.0: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -689,24 +449,12 @@ rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.5
-rule "PII.5.0: Redact line after contact information keywords reduced"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)"
+rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -716,10 +464,10 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)"
+rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -729,23 +477,12 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: Redact line between contact keywords"
-    when
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -754,10 +491,10 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -766,41 +503,12 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.3", "Found between contact keywords", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.7
-rule "PII.7.0: Redact contact information if applicant is found"
-    when
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -815,10 +523,10 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -833,13 +541,14 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                           containsStringIgnoreCase("producer of the active substance") ||
                           containsStringIgnoreCase("manufacturer of the active substance") ||
@@ -855,25 +564,7 @@ rule "PII.8.0: Redact contact information if producer is found"
             .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -888,25 +579,27 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -963,66 +656,38 @@ rule "ETC.0.0: Purity Hint"
 
 
 // Rule unit: ETC.2
-rule "ETC.2.0: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
+rule "ETC.2.0: Redact signatures (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "ETC.2.2: Redact signatures (vertebrate study)"
+rule "ETC.2.1: Redact signatures (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.3: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.3", "Signature Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.3.1: Skip logos (non vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.skip("ETC.3.1", "Logo Found");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.2: Redact logos (vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.3.3: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -1037,7 +702,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
@@ -1229,7 +893,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -1237,6 +901,22 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
     end
 
 
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
@@ -1344,15 +1024,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1312,7 +1314,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -1414,15 +1416,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -1,953 +0,0 @@
-package drools
-
-import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.Collection;
-import java.util.stream.Stream;
-import java.util.Optional;
-
-import com.iqser.red.service.redaction.v1.server.model.document.*;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-
-global Document document
-global EntityCreationService entityCreationService
-global ManualChangesApplicationService manualChangesApplicationService
-global Dictionary dictionary
-
-//------------------------------------ queries ------------------------------------
-
-query "getFileAttributes"
-    $fileAttribute: FileAttribute()
-    end
-
-//------------------------------------ Syngenta specific rules ------------------------------------
-
-// Rule unit: SYN.1
-rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
-    when
-        $section: Section(containsString("CT") || containsString("BL"))
-    then
-        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
-        entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("SYN.1.0", ""));
-    end
-
-
-//------------------------------------ CBI rules ------------------------------------
-
-// Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
-    then
-        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
-    then
-        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: CBI.1
-rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
-    then
-        $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
-    end
-
-rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
-    then
-        $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: CBI.2
-rule "CBI.2.0: Do not redact genitive CBI Author"
-    when
-        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
-    then
-        entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
-        .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
-    end
-
-
-// Rule unit: CBI.7
-rule "CBI.7.0: Do not redact Names and Addresses if published information found in Section without tables"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("published_information"),
-                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.0",
-                        "Published Information found in section",
-                        $section.getEntitiesOfType("published_information")
-                );
-            });
-    end
-
-rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
-    when
-        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
-        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
-        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
-    then
-        $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
-    end
-
-
-// Rule unit: CBI.9
-rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: CBI.10
-rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: CBI.11
-rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -1
-    when
-        $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
-    then
-        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
-    end
-
-
-// Rule unit: CBI.16
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-
-// Rule unit: CBI.17
-rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
-    when
-        $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
-    then
-        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
-    end
-
-rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
-    when
-        $section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
-    then
-        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
-    end
-
-
-// Rule unit: CBI.20
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-//------------------------------------ PII rules ------------------------------------
-
-// Rule unit: PII.0
-rule "PII.0.1: Redact all PII (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.2: Redact all PII (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: PII.1
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.3: Redact typoed Emails with indicator"
-    when
-        $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
-    then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.2
-rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.3: Redact phone numbers without indicators"
-    when
-        $section: Section(containsString("+"))
-    then
-        entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section)
-            .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.3
-rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.7
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.8
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.10
-rule "PII.10.0: Redact study director abbreviation"
-    when
-        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
-    then
-        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-     end
-
-
-// Rule unit: PII.11
-rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
-    when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
-    then
-        entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.12
-rule "PII.12.0: Expand PII entities with salutation prefix"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
-    then
-        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
-        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.12.1: Expand PII entities with salutation prefix"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
-    then
-        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
-        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-//------------------------------------ Other rules ------------------------------------
-
-// Rule unit: ETC.0
-rule "ETC.0.0: Purity Hint"
-    when
-        $section: Section(containsStringIgnoreCase("purity"))
-    then
-        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
-            .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
-    end
-
-
-// Rule unit: ETC.2
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.2: Redact signatures (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: ETC.3
-rule "ETC.3.1: Skip logos (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.skip("ETC.3.1", "Logo Found");
-    end
-
-rule "ETC.3.2: Redact logos (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: ETC.5
-rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
-    when
-        FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type() == "dossier_redaction")
-    then
-        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
-    when
-        not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type() == "dossier_redaction")
-    then
-        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
-        retract($dossierRedaction);
-    end
-
-
-//------------------------------------ AI rules ------------------------------------
-
-// Rule unit: AI.0
-rule "AI.0.0: Add all NER Entities of type CBI_author"
-    salience 999
-    when
-        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
-    then
-        nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
-    end
-
-
-// Rule unit: AI.1
-rule "AI.1.0: Combine and add NER Entities as CBI_address"
-    salience 999
-    when
-        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
-    then
-        entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
-    end
-
-
-//------------------------------------ Manual changes rules ------------------------------------
-
-// Rule unit: MAN.0
-rule "MAN.0.0: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $entityToBeResized: TextEntity(matchesAnnotationId($id))
-    then
-        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($entityToBeResized);
-        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.0.1: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $imageToBeResized: Image(id == $id)
-    then
-        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($imageToBeResized);
-        update($imageToBeResized.getParent());
-    end
-
-
-// Rule unit: MAN.1
-rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
-        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($entityToBeRemoved);
-        retract($idRemoval);
-        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId)
-        $imageEntityToBeRemoved: Image($id == id)
-    then
-        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($imageEntityToBeRemoved);
-        retract($idRemoval);
-        update($imageEntityToBeRemoved.getParent());
-    end
-
-
-// Rule unit: MAN.2
-rule "MAN.2.0: Apply force redaction"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId)
-        $entityToForce: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToForce.getManualOverwrite().addChange($force);
-        update($entityToForce);
-        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
-        retract($force);
-    end
-
-rule "MAN.2.1: Apply force redaction to images"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId)
-        $imageToForce: Image(id == $id)
-    then
-        $imageToForce.getManualOverwrite().addChange($force);
-        update($imageToForce);
-        update($imageToForce.getParent());
-        retract($force);
-    end
-
-
-// Rule unit: MAN.3
-rule "MAN.3.0: Apply entity recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
-    then
-        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        update($entityToBeRecategorized);
-        retract($recategorization);
-    end
-
-rule "MAN.3.1: Apply entity recategorization of same type"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
-    then
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        retract($recategorization);
-    end
-
-rule "MAN.3.2: Apply image recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
-    then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
-        retract($recategorization);
-    end
-
-rule "MAN.3.3: Apply recategorization entities by default"
-    salience 128
-    when
-        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
-    then
-        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
-    end
-
-
-// Rule unit: MAN.4
-rule "MAN.4.0: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
-        $imageToBeRecategorized: Image($id == id)
-    then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
-        update($imageToBeRecategorized)
-        retract($legalBasisChange)
-    end
-
-rule "MAN.4.1: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
-        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
-        update($entityToBeChanged)
-        retract($legalBasisChange)
-    end
-
-
-//------------------------------------ Entity merging rules ------------------------------------
-
-// Rule unit: X.0
-rule "X.0.0: Remove Entity contained by Entity of same type"
-    salience 65
-    when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
-    then
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
-        retract($contained);
-    end
-
-
-// Rule unit: X.2
-rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
-    salience 64
-    when
-        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-    then
-        $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
-        retract($entity)
-    end
-
-
-// Rule unit: X.3
-rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
-    salience 64
-    when
-        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
-    salience 256
-    when
-        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
-    salience 256
-    when
-        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
-        retract($recommendation);
-    end
-
-rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
-    salience 256
-    when
-        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
-    salience 32
-    when
-        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
-    then
-        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
-        retract($lowerRank);
-    end
-
-rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
-    salience 32
-    when
-         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
-    then
-        $inner.getIntersectingNodes().forEach(node -> update(node));
-        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
-        retract($inner);
-    end
-
-
-// Rule unit: X.8
-rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
-    salience 257
-    when
-        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
-        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
-    then
-        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
-        $entity.addEngines($other.getEngines());
-        retract($other);
-    end
-
-rule "X.8.1: Remove Entity when intersected by imported Entity"
-    salience 256
-    when
-        $entity: TextEntity(engines contains Engine.IMPORTED, active())
-        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
-    then
-        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
-        retract($other);
-    end
-
-
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
-    when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
-    then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
-    end
-
-
-//------------------------------------ File attributes rules ------------------------------------
-
-// Rule unit: FA.1
-rule "FA.1.0: Remove duplicate FileAttributes"
-    salience 64
-    when
-        $fileAttribute: FileAttribute($label: label, $value: value)
-        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
-    then
-        retract($duplicate);
-    end
-
-
-//------------------------------------ Local dictionary search rules ------------------------------------
-
-// Rule unit: LDS.0
-rule "LDS.0.0: Run local dictionary search"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -999
-    when
-        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
-    then
-        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
-            .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
-                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
-            });
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -255,15 +257,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -100,18 +102,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -150,23 +151,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -185,7 +184,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $section.getEntitiesOfType("redaction_indicator").stream(),
+                            $section.getEntitiesOfType("vertebrate").stream(),
                             $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
                 );
             });
@@ -193,22 +192,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -230,18 +227,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                       "CBI.8.1",
-                       "Must_redact found",
-                       "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                       $table.getEntitiesOfTypeInSameRow("must_redact", entity)
-               );
-            });
+        $authorOrAddress.applyWithReferences(
+                        "CBI.8.1",
+                        "Must_redact found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
+                );
     end
 
 
@@ -272,19 +268,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
             .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.11
 rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
@@ -298,22 +281,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
 
 
 // Rule unit: CBI.12
-rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
-        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
-        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
-        $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
-        TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
-    then
-        entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT)
-            .ifPresent(yesEntity -> {
-                yesEntity.skip("CBI.12.0", "must_redact");
-            });
-    end
-
-rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
+rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -325,13 +293,13 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
     then
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
-              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
               dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
           });
 
     end
 
-rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
+rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -344,13 +312,13 @@ rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s
 
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
-              authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
               dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
           });
 
     end
 
-rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
+rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
     when
         $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
         TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
@@ -359,19 +327,16 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
         $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
     then
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
-          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
+          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
 
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $section: Section(containsStringIgnoreCase("batches produced at"))
         $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section)
-            .forEach(entity -> entity.skip("CBI.14.0", "must_redact"));
     end
 
 
@@ -392,10 +357,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
                           containsStringIgnoreCase($residueKeyword),
                           containsStringIgnoreCase($keyword))
     then
-        entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section)
-            .forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
+        entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
+            .toList();
 
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+        $section.getEntitiesOfType(List.of($keyword, $residueKeyword))
             .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
@@ -413,8 +378,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
         $residueKeyword: String() from List.of("determination of residues", "determination of total residues")
         $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword))
     then
-        entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table)
-            .forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
+        entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
+            .toList();
 
         $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
             .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
@@ -423,19 +388,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -443,12 +396,12 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -456,19 +409,7 @@ rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -526,19 +467,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
 
 
 // Rule unit: CBI.20
-rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
+rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -546,12 +475,12 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
+             laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
+rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -559,117 +488,56 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\""
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.3", "PERFORMING LABORATORY was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.0: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "PII.0.2: Redact all PII (vertebrate study)"
+rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.3: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.3", "Personal Information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: PII.1
-rule "PII.1.0: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
+rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
+rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.5: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.5", "Found by Email Regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.4
-rule "PII.4.0: Redact line after contact information keywords"
+rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -695,62 +563,7 @@ rule "PII.4.0: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -775,23 +588,12 @@ rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: Redact line between contact keywords"
-    when
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -800,10 +602,10 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -812,41 +614,12 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.3", "Found between contact keywords", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.7
-rule "PII.7.0: Redact contact information if applicant is found"
-    when
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -861,10 +634,10 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -879,13 +652,14 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                           containsStringIgnoreCase("producer of the active substance") ||
                           containsStringIgnoreCase("manufacturer of the active substance") ||
@@ -901,25 +675,7 @@ rule "PII.8.0: Redact contact information if producer is found"
             .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -934,25 +690,27 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -999,66 +757,38 @@ rule "ETC.1.0: Redact Purity"
 
 
 // Rule unit: ETC.2
-rule "ETC.2.0: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
+rule "ETC.2.0: Redact signatures (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "ETC.2.2: Redact signatures (vertebrate study)"
+rule "ETC.2.1: Redact signatures (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.3: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.3", "Signature Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.3.1: Skip logos (non vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.skip("ETC.3.1", "Logo Found");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.2: Redact logos (vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.3.3: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -1082,7 +812,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
@@ -1317,7 +1046,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -1325,6 +1054,22 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
     end
 
 
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
@@ -1432,15 +1177,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -80,12 +82,12 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 
@@ -248,7 +250,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -256,6 +258,22 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
     end
 
 
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
@@ -363,15 +381,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -374,7 +376,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -500,15 +502,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -274,7 +276,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -282,6 +284,22 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
     end
 
 
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
@@ -400,6 +418,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml

@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>

redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml

@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -459,7 +459,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -33,7 +33,11 @@ public class RuleFileMigrator {
             rulesToAdd.forEach(ruleFileBluePrint::addRule);
         }
 
-        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
+        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
+                                                               combinedBluePrint.globals(),
+                                                               combinedBluePrint.queries(),
+                                                               ruleFileBluePrint.ruleClasses());
+        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
         String migratedFilePath = ruleFile.getAbsolutePath();
         try (var out = new FileOutputStream(migratedFilePath)) {
             out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1299,7 +1301,6 @@ rule "TAB.7.0: Indicator (Species)"
             .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
     end
 
-
 //------------------------------------ Manual redaction rules ------------------------------------
 
 // Rule unit: MAN.0
@@ -1458,7 +1459,7 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@@ -1561,18 +1562,26 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
-
-// Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
-    salience 64
+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
     when
-        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
     end
 
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 
 //------------------------------------ File attributes rules ------------------------------------
 

redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java

@@ -25,9 +25,9 @@ public class RuleFileMigrationTest {
 
     // Put your redaction service drools paths and dossier-templates paths both RM and DM here
     static final List<String> ruleFileDirs = List.of(
-            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            "/home/kschuettler/iqser/fforesight/dossier-templates-v2",
-            "/home/kschuettler/iqser/redaction/dossier-templates-v2");
+            "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
+            "/Users/maverickstuder/Documents/DocuMine/dossier-templates-v2/",
+            "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2/");
 
 
     @Test
@@ -36,11 +36,7 @@ public class RuleFileMigrationTest {
     void migrateAllEntityRules() {
 
         for (String ruleFileDir : ruleFileDirs) {
-            Files.walk(Path.of(ruleFileDir))
-                    .filter(this::isEntityRuleFile)
-                    .map(Path::toFile)
-                    .peek(System.out::println)
-                    .forEach(RuleFileMigrator::migrateFile);
+            Files.walk(Path.of(ruleFileDir)).filter(this::isEntityRuleFile).map(Path::toFile).peek(System.out::println).forEach(RuleFileMigrator::migrateFile);
         }
     }