test publish stuff

RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature

Closes RED-8590

See merge request redactmanager/redaction-service!293

.gitlab-ci.yml

@@ -21,3 +21,16 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+
+pages:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
+    - mkdir public
+    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+  artifacts:
+    paths:
+      - public

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.338.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DroolsSyntaxDeprecatedWarnings {
+
+    Integer line;
+    Integer column;
+    String message;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -19,6 +19,8 @@ public class DroolsSyntaxValidation {
 
     @Builder.Default
     List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
+    @Builder.Default
+    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
 
 
     public void addErrorMessage(int line, int column, String message) {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,15 +1,11 @@
 package com.iqser.red.service.redaction.v1.model;
 
-import java.util.Collections;
-import java.util.Set;
-
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.NonNull;
 
 @Data
 @Builder
@@ -17,16 +13,9 @@ import lombok.NonNull;
 @AllArgsConstructor
 public class MigrationRequest {
 
-    @NonNull
     String dossierTemplateId;
-    @NonNull
     String dossierId;
-    @NonNull
     String fileId;
-    @NonNull
     ManualRedactions manualRedactions;
-    @NonNull
-    @Builder.Default
-    Set<String> entitiesWithComments = Collections.emptySet();
 
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.89.3"
+val layoutParserVersion = "0.94.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.349.26"
+val persistenceServiceVersion = "2.360.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {
@@ -60,6 +60,8 @@ dependencies {
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
 
+    implementation("org.reflections:reflections:0.10.2")
+
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -82,9 +84,6 @@ tasks.test {
     maxHeapSize = "1024m"
 }
 
-
-
-
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -111,3 +110,40 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         tags.set(listOf(dockerTag))
     }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -46,10 +46,18 @@ public class Application {
         return new TimedAspect(registry);
     }
 
+
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
+
+    @Bean(initMethod = "findDeprecatedMethods")
+    public DeprecatedElementsFinder deprecatedElementsFinder() {
+
+        return new DeprecatedElementsFinder();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.ConfigurationBuilder;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DeprecatedElementsFinder {
+
+    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    private Set<Method> deprecatedMethods;
+    @Getter
+    private Map<String, String> deprecatedMethodsSignaturesMap;
+    private Set<Class<?>> deprecatedClasses;
+    @Getter
+    private SearchImplementation classesSearchImplementation;
+    @Getter
+    private SearchImplementation methodsSearchImplementation;
+
+
+    public void findDeprecatedMethods() {
+
+        deprecatedMethods = new HashSet<>();
+        deprecatedClasses = new HashSet<>();
+        deprecatedMethodsSignaturesMap = new HashMap<>();
+
+        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
+                                                          .setExpandSuperTypes(true)
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+
+        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
+
+        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
+
+        if (!deprecatedClasses.isEmpty()) {
+            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
+                                                                                         .collect(Collectors.toSet())));
+            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
+                                                                           .map(Class::getName)
+                                                                           .collect(Collectors.toSet()), false);
+        }
+
+        if (!deprecatedMethods.isEmpty()) {
+            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
+                                                                           .map(Method::getName)
+                                                                           .collect(Collectors.toSet()), false);
+            deprecatedMethodsSignaturesMap = new HashMap<>();
+            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
+                    .collect(groupingBy(Method::getName));
+            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
+                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
+                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
+                                                                       .stream()
+                                                                       .map(this::getMethodSignature)
+                                                                       .collect(Collectors.joining("\n")));
+            }
+
+        }
+
+        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
+    }
+
+
+    public Set<Method> getDeprecatedMethods() {
+
+        if (this.deprecatedMethods == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedMethods;
+    }
+
+
+    public Set<Class<?>> getDeprecatedClasses() {
+
+        if (this.deprecatedClasses == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedClasses;
+    }
+
+    private String getMethodSignature(Method method) {
+
+        String methodName = method.getName();
+        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
+                .map(Type::getTypeName)
+                .collect(Collectors.joining(", "));
+
+        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
@@ -93,16 +92,11 @@ public class LegacyRedactionLogMergeService {
         return redactionLog;
     }
 
-
     public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
 
-        return createManualRedactionWrappers(manualRedactions).stream()
-                .map(ManualRedactionWrapper::getId)
-                .distinct()
-                .count();
+        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
     }
 
-
     private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
 
         List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
@@ -203,12 +197,7 @@ public class LegacyRedactionLogMergeService {
         }
 
         redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization)
-                             .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                             .withChange("type", imageRecategorization.getType())
-                             .withChange("section", imageRecategorization.getSection())
-                             .withChange("legalBasis", imageRecategorization.getLegalBasis())
-                             .withChange("value", imageRecategorization.getValue()));
+                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
     }
 
 
@@ -380,17 +369,13 @@ public class LegacyRedactionLogMergeService {
     @SuppressWarnings("PMD.UselessParentheses")
     private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
 
-        if (manualRedactionEntry.getDictionaryEntryType() != null //
-            && (manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE) //
-                || manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_RECOMMENDATION))) {
+        if (!manualRedactionEntry.isApproved()) {
             return false;
         }
 
-        if (manualRedactionEntry.getProcessedDate() == null) {
-            return false;
-        }
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary());
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
+                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
+                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -57,7 +57,7 @@ public class MigrationMessageReceiver {
 
         if (redactionLog.getAnalysisVersion() == 0) {
             redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else {
+        } else if (migrationRequest.getManualRedactions() != null) {
             redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
                                                                                                       migrationRequest.getManualRedactions(),
                                                                                                       migrationRequest.getDossierTemplateId());
@@ -67,8 +67,7 @@ public class MigrationMessageReceiver {
                                                                                               document,
                                                                                               migrationRequest.getDossierTemplateId(),
                                                                                               migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId(),
-                                                                                              migrationRequest.getEntitiesWithComments());
+                                                                                              migrationRequest.getFileId());
 
         log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -8,7 +8,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -35,6 +34,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
@@ -54,16 +54,12 @@ public class RedactionLogToEntityLogMigrationService {
 
     private static final double MATCH_THRESHOLD = 10;
     EntityFindingUtility entityFindingUtility;
+    EntityEnrichmentService entityEnrichmentService;
     DictionaryService dictionaryService;
     ManualChangesApplicationService manualChangesApplicationService;
 
 
-    public MigratedEntityLog migrate(RedactionLog redactionLog,
-                                     Document document,
-                                     String dossierTemplateId,
-                                     ManualRedactions manualRedactions,
-                                     String fileId,
-                                     Set<String> entitiesWithComments) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
 
         log.info("Migrating entities for file {}", fileId);
         List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
@@ -71,8 +67,8 @@ public class RedactionLogToEntityLogMigrationService {
         MigratedIds migratedIds = entitiesToMigrate.stream()
                 .collect(new MigratedIdsCollector());
 
+        applyManualChanges(entitiesToMigrate, manualRedactions);
         log.info("applying manual changes to migrated entities for file {}", fileId);
-        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -93,7 +89,7 @@ public class RedactionLogToEntityLogMigrationService {
                                             .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
                                             .toList());
 
-        if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
+        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
             String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
                                            redactionLog.getRedactionLogEntry().size(),
                                            entityLog.getEntityLogEntry().size());
@@ -101,14 +97,8 @@ public class RedactionLogToEntityLogMigrationService {
             throw new AssertionError(message);
         }
 
-        Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
-                .stream()
-                .filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
-                .map(BaseAnnotation::getAnnotationId)
-                .collect(Collectors.toSet());
-
         MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
+                .filter(MigrationEntity::hasManualChangesOrComments)
                 .filter(m -> !m.getOldId().equals(m.getNewId()))
                 .collect(new MigratedIdsCollector());
 
@@ -123,7 +113,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
+    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return;
@@ -136,8 +126,6 @@ public class RedactionLogToEntityLogMigrationService {
                                                                                    manualRedactions.getLegalBasisChanges(),
                                                                                    manualRedactions.getRecategorizations())
                 .flatMap(Collection::stream)
-                .filter(manualChange -> manualChange.getProcessedDate() != null)
-                .filter(BaseAnnotation::isLocal)
                 .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
 
         entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
@@ -147,18 +135,15 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
+    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
 
-        return redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
-                .count();
+        return redactionLog.getRedactionLogEntry().size();
     }
 
 
     private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
         List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
         return Stream.of(textMigrationEntities.stream(), images.stream())
                 .flatMap(Function.identity())
@@ -172,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
 
         List<Image> images = document.streamAllImages()
                 .collect(Collectors.toList());
@@ -219,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
             } else {
                 closestImage.skip(ruleIdentifier, reason);
             }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
         }
         return migrationEntities;
     }
@@ -265,8 +250,7 @@ public class RedactionLogToEntityLogMigrationService {
         List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
                 .toList();
 
         List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
@@ -303,20 +287,4 @@ public class RedactionLogToEntityLogMigrationService {
         return entitiesToMigrate;
     }
 
-
-    private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
-
-        var pages = redactionLogEntry.getPositions()
-                .stream()
-                .map(Rectangle::getPage)
-                .collect(Collectors.toSet());
-
-        for (int page : pages) {
-            if (page > numberOfPages) {
-                return false;
-            }
-        }
-        return true;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -4,11 +4,9 @@ import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreator
 import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
 
 import java.awt.geom.Rectangle2D;
-import java.time.OffsetDateTime;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -20,10 +18,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
@@ -32,8 +28,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
@@ -52,8 +46,6 @@ public final class MigrationEntity {
 
     private final PrecursorEntity precursorEntity;
     private final RedactionLogEntry redactionLogEntry;
-    private final DictionaryService dictionaryService;
-    private final String dossierTemplateId;
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
@@ -63,10 +55,10 @@ public final class MigrationEntity {
     List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
 
-        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
         PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
+
         if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
             precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         } else if (redactionLogEntry.lastChangeIsRemoved()) {
@@ -81,32 +73,13 @@ public final class MigrationEntity {
             precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         }
 
-        return MigrationEntity.builder()
-                .precursorEntity(precursorEntity)
-                .redactionLogEntry(redactionLogEntry)
-                .oldId(redactionLogEntry.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
+        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
     }
 
 
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
-                                                        Image image,
-                                                        String fileId,
-                                                        DictionaryService dictionaryService,
-                                                        String dossierTemplateId) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
 
-        return MigrationEntity.builder()
-                .redactionLogEntry(redactionLogImage)
-                .migratedEntity(image)
-                .oldId(redactionLogImage.getId())
-                .newId(image.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
+        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
     }
 
 
@@ -183,6 +156,18 @@ public final class MigrationEntity {
     }
 
 
+    private static EntryType getEntryType(EntityType entityType) {
+
+        return switch (entityType) {
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
+            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
+            case RECOMMENDATION -> EntryType.RECOMMENDATION;
+            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+        };
+    }
+
+
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -205,6 +190,9 @@ public final class MigrationEntity {
         entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
         entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
         entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
+        if (redactionLogEntry.getLegalBasis() != null) {
+            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        }
 
         if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
             entityLogEntry.setState(EntryState.IGNORED);
@@ -237,15 +225,13 @@ public final class MigrationEntity {
 
     public EntityLogEntry createEntityLogEntry(Image image) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
         return EntityLogEntry.builder()
                 .id(image.getId())
-                .value(image.getValue())
-                .type(imageType)
+                .value(image.value())
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(false)
                 .positions(positions)
@@ -257,7 +243,7 @@ public final class MigrationEntity {
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
                 .state(buildEntryState(image))
-                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
@@ -268,8 +254,7 @@ public final class MigrationEntity {
         return EntityLogEntry.builder()
                 .id(precursorEntity.getId())
                 .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(precursorEntity.legalBasis())
                 .value(precursorEntity.value())
                 .type(precursorEntity.type())
                 .state(buildEntryState(precursorEntity))
@@ -303,8 +288,7 @@ public final class MigrationEntity {
                 .id(entity.getId())
                 .positions(rectanglesPerLine)
                 .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue()
                                .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                 .type(entity.type())
@@ -347,11 +331,11 @@ public final class MigrationEntity {
     }
 
 
-    public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
+    public boolean hasManualChangesOrComments() {
 
         return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
                !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
+               || hasManualChanges();
     }
 
 
@@ -366,11 +350,17 @@ public final class MigrationEntity {
         manualChanges.addAll(manualChangesToApply);
         manualChangesToApply.forEach(manualChange -> {
             if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
-                manualResizeRedaction.setAnnotationId(newId);
-                manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
-            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
-                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
-                migratedEntity.getManualOverwrite().addChange(manualChange);
+                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
+                // Instead, we add an add_locally change to the db.
+                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
+                        .positions(manualResizeRedaction.getPositions())
+                        .annotationId(getNewId())
+                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
+                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
+                        .textAfter(manualResizeRedaction.getTextAfter())
+                        .textBefore(manualResizeRedaction.getTextBefore())
+                        .build();
+                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
             } else {
                 migratedEntity.getManualOverwrite().addChange(manualChange);
             }
@@ -388,25 +378,19 @@ public final class MigrationEntity {
                 .findFirst()
                 .orElse(manualChanges.get(0)).getUser();
 
-        OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
-
         return ManualRedactionEntry.builder()
                 .annotationId(newId)
                 .fileId(fileId)
-                .user(user)
-                .requestDate(requestDate)
                 .type(redactionLogEntry.getType())
                 .value(redactionLogEntry.getValue())
                 .reason(redactionLogEntry.getReason())
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .section(redactionLogEntry.getSection())
-                .rectangle(false)
                 .addToDictionary(false)
                 .addToDossierDictionary(false)
+                .rectangle(false)
                 .positions(buildPositions(migratedEntity))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .dictionaryEntryType(DictionaryEntryType.ENTRY)
+                .user(user)
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -129,12 +129,8 @@ public class ManualChangeOverwrite {
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
-                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
-                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
-                section = recategorization.getSection();
-                value = recategorization.getValue();
                 if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
                     legalBasis = recategorization.getLegalBasis();
                 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,6 +14,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -33,6 +34,8 @@ public class Document implements GenericSemanticNode {
 
     @EqualsAndHashCode.Include
     List<Integer> treeId = Collections.emptyList();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     Set<Page> pages;
     DocumentTree documentTree;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Footer implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Header implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -35,6 +36,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
@@ -94,7 +98,7 @@ public class Image implements GenericSemanticNode, IEntity {
     public String type() {
 
         return getManualOverwrite().getType()
-                .orElse(imageType.toString().toLowerCase(Locale.ENGLISH));
+                .orElse(imageType.toString());
     }
 
 
@@ -127,6 +131,32 @@ public class Image implements GenericSemanticNode, IEntity {
     }
 
 
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if(bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
+        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+        return (area / calculatedIntersection) > containmentThreshold;
+    }
+
+
     public int length() {
 
         return 0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java

@@ -6,10 +6,22 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER,
+    OTHER {
+        @Override
+        public String toString() {
+
+            return "image";
+        }
+    },
     OCR;
 
 
+    public String toString() {
+
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -10,12 +10,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
 
@@ -37,6 +40,9 @@ public class Section implements GenericSemanticNode {
 
     Map<Page, Rectangle2D> bBoxCache;
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
 
     @Override
     public NodeType getType() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -356,6 +357,13 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
+    Set<LayoutEngine> getEngines();
+
+    default void addEngine(LayoutEngine engine) {
+        getEngines().add(engine);
+    }
+
+
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -554,7 +562,6 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
-
             textEntity.addIntersectingNode(this);
             streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -32,6 +33,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -12,7 +13,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -38,11 +41,9 @@ public class EntityChangeLogService {
         boolean hasChanges = false;
 
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-
             Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
                     .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                     .findAny();
-
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -50,18 +51,14 @@ public class EntityChangeLogService {
             }
 
             EntityLogEntry previousEntity = optionalPreviousEntity.get();
-
             entityLogEntry.getChanges().addAll(previousEntity.getChanges());
-
             if (!previousEntity.getState().equals(entityLogEntry.getState())) {
                 hasChanges = true;
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
                 entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
             }
         }
-
         addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
-
         return hasChanges;
     }
 
@@ -75,20 +72,36 @@ public class EntityChangeLogService {
         Set<String> existingIds = newEntityLogEntries.stream()
                 .map(EntityLogEntry::getId)
                 .collect(Collectors.toSet());
-
         List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                 .filter(entry -> !existingIds.contains(entry.getId()))
+                .collect(Collectors.toList());
+        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
+                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
                 .toList();
-
-        removedEntries.stream()
-                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
-                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
-                .forEach(entry -> entry.setState(EntryState.REMOVED));
-
+        previousEntityLogEntries.removeAll(removedDossierRedaction);
+        removedEntries.removeAll(removedDossierRedaction);
+        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
+        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
         newEntityLogEntries.addAll(removedEntries);
     }
 
 
+    private void addManualChangeForDictionaryRemovals(EntityLogEntry entry, ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null || manualRedactions.getIdsToRemove().isEmpty()) {
+            return;
+        }
+
+        manualRedactions.getIdsToRemove()
+                .stream()
+                .filter(IdRemoval::isRemoveFromDictionary)//
+                .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
+                .findFirst()//
+                .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
+    }
+
+
     private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
         if (state.equals(previousState)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -186,12 +186,11 @@ public class EntityLogCreatorService {
 
     private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
-                .type(imageType)
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -30,12 +30,8 @@ public class ManualChangeFactory {
     public ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
 
         ManualChange manualChange = ManualChange.from(baseAnnotation);
-        if (baseAnnotation instanceof ManualRecategorization recategorization) {
-            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                    .withChange("type", recategorization.getType())
-                    .withChange("section", recategorization.getSection())
-                    .withChange("legalBasis", recategorization.getLegalBasis())
-                    .withChange("value", recategorization.getValue());
+        if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
+            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
             manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
         } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -6,6 +6,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
@@ -76,6 +77,7 @@ public class DocumentGraphMapper {
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -590,9 +590,6 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        if (trimmedTextRange.length() == 0){
-            return Optional.empty();
-        }
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
         if (node.getEntities().contains(entity)) {
             Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
@@ -664,8 +661,6 @@ public class EntityCreationService {
 
         addEntityToGraph(mergedEntity, node);
         insertToKieSession(mergedEntity);
-
-        entitiesToMerge.stream().filter(e -> !e.equals(mergedEntity)).forEach(node.getEntities()::remove);
         return mergedEntity;
     }
 
@@ -749,38 +744,25 @@ public class EntityCreationService {
         try {
             if (node.getEntities().contains(entity)) {
                 // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
-                Optional<TextEntity> optionalTextEntity = node.getEntities()
-                        .stream()//
+                node.getEntities().stream()//
                         .filter(e -> e.equals(entity))//
                         .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
-                        .findAny();
-                if (optionalTextEntity.isPresent()) {
-                    addDuplicateEntityToGraph(optionalTextEntity.get(), entity.getTextRange(), node);
-                } else {
-                    node.getEntities().remove(entity);
-                    addNewEntityToGraph(entity, documentTree);
-                }
-
+                        .findAny()//
+                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
             } else {
                 entity.addIntersectingNode(documentTree.getRoot().getNode());
                 addEntityToGraph(entity, documentTree);
             }
         } catch (NoSuchElementException e) {
-            addNewEntityToGraph(entity, documentTree);
+            entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
+            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
+            entity.addIntersectingNode(documentTree.getRoot().getNode());
+            addToPages(entity);
+            addEntityToNodeEntitySets(entity);
         }
     }
 
 
-    private void addNewEntityToGraph(TextEntity entity, DocumentTree documentTree) {
-
-        entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
-        entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
-        entity.addIntersectingNode(documentTree.getRoot().getNode());
-        addToPages(entity);
-        addEntityToNodeEntitySets(entity);
-    }
-
-
     private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
 
         entityToDuplicate.addTextRange(newTextRange);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -47,9 +47,7 @@ public class EntityFindingUtility {
     }
 
 
-    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
-                                                                          Map<String, List<TextEntity>> entitiesWithSameValue,
-                                                                          double matchThreshold) {
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
 
         if (precursorEntity.getValue() == null) {
             return Optional.empty();
@@ -58,7 +56,7 @@ public class EntityFindingUtility {
         List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
 
         if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
+            log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
             return Optional.empty();
         }
 
@@ -68,22 +66,18 @@ public class EntityFindingUtility {
                 .min(Comparator.comparingDouble(ClosestEntity::getDistance));
 
         if (optionalClosestEntity.isEmpty()) {
-            log.info("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
+            log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
             return Optional.empty();
         }
 
         ClosestEntity closestEntity = optionalClosestEntity.get();
         if (closestEntity.getDistance() > matchThreshold) {
-            log.info("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
-                     precursorEntity.getValue(),
-                     precursorEntity.getEntityPosition()
-                             .get(0).pageNumber(),
-                     precursorEntity.getEntityPosition()
-                             .stream()
-                             .map(RectangleWithPage::rectangle2D)
-                             .toList(),
-                     closestEntity.getDistance(),
-                     matchThreshold);
+            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                    precursorEntity.getValue(),
+                    precursorEntity.getEntityPosition().get(0).pageNumber(),
+                    precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    closestEntity.getDistance(),
+                    matchThreshold);
             return Optional.empty();
         }
 
@@ -99,14 +93,8 @@ public class EntityFindingUtility {
 
     private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
 
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage()
-                .stream()
-                .map(PositionOnPage::getPage)
-                .map(Page::getNumber)
-                .collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream()
-                .map(RectangleWithPage::pageNumber)
-                .collect(Collectors.toSet());
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
         return entityPageNumbers.containsAll(originalPageNumbers);
     }
 
@@ -117,16 +105,15 @@ public class EntityFindingUtility {
             return Double.MAX_VALUE;
         }
         return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .average()
                 .orElse(Double.MAX_VALUE);
     }
 
 
     private static long countRectangles(TextEntity entity) {
 
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
     }
 
 
@@ -174,8 +161,7 @@ public class EntityFindingUtility {
                     pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
                     node.getPages()));
         }
-
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues.stream().map(String::trim).collect(Collectors.toSet()), true);
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -9,6 +9,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@@ -22,21 +23,29 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
 import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-@RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityFromPrecursorCreationService {
 
     static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
     EntityFindingUtility entityFindingUtility;
+    EntityCreationService entityCreationService;
     DictionaryService dictionaryService;
 
 
+    @Autowired
+    public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
+
+        this.entityFindingUtility = entityFindingUtility;
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
     public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
 
         Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -57,18 +57,23 @@ public class SectionFinderService {
                 if (entry.getContainingNodeId().isEmpty()) {
                     continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
                 }
-                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
+                sectionsToReanalyse.add(entry.getContainingNodeId()
+                                                .get(0));
             }
         }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
+                                                                          .stream()
+                                                                          .map(DictionaryIncrementValue::getValue)
+                                                                          .toList(), true);
 
-        document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
-            }
-        });
+        document.streamChildren()
+                .forEach(mainNode -> {
+                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                        sectionsToReanalyse.add(mainNode.getTreeId()
+                                                        .get(0));
+                    }
+                });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
         relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
@@ -87,8 +92,11 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId().get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody()
+                                                       .stream()
+                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId()
+                        .get(0))
                 .toList();
     }
 
@@ -99,8 +107,7 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions()
-                .getEntriesToAdd()
+        return analyzeRequest.getManualRedactions().getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -132,11 +139,22 @@ public class SectionFinderService {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                                     .stream()
+                                     .map(ManualResizeRedaction::getAnnotationId),
+                             Stream.concat(manualRedactions.getLegalBasisChanges()
+                                                   .stream()
+                                                   .map(ManualLegalBasisChange::getAnnotationId),
+                                           Stream.concat(manualRedactions.getRecategorizations()
+                                                                 .stream()
+                                                                 .map(ManualRecategorization::getAnnotationId),
+                                                         Stream.concat(manualRedactions.getIdsToRemove()
+                                                                               .stream()
+                                                                               .map(IdRemoval::getAnnotationId),
+                                                                       manualRedactions.getForceRedactions()
+                                                                               .stream()
+                                                                               .map(ManualForceRedaction::getAnnotationId)))))
+                .collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -12,24 +15,34 @@ import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
+import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DroolsSyntaxValidationService {
-    
+
     private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
+    private final DeprecatedElementsFinder deprecatedElementsFinder;
+
 
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
@@ -43,15 +56,74 @@ public class DroolsSyntaxValidationService {
         }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
+        droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings());
         return droolsCompilerSyntaxValidation;
     }
 
+    private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
+
+        if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
+            String imports = ruleFileBluePrint.getImports();
+            SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation();
+            List<SearchImplementation.MatchPosition> matches = classesSearchImplementation.getMatches(imports);
+            if (!matches.isEmpty()) {
+                String sb = "Following imports are deprecated: \n" + matches.stream()
+                        .map(m -> imports.substring(m.startIndex(), m.endIndex()))
+                        .collect(Collectors.joining("\n"));
+                return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb)
+                                            .build();
+            }
+        }
+        return null;
+    }
+
+    private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
+
+        List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
+
+        // checks the rules for deprecateMethods
+        if (!deprecatedElementsFinder.getDeprecatedMethods().isEmpty()) {
+            SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation();
+            Map<String, String> deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap();
+
+            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
+                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
+                    for (BasicRule basicRule : ruleUnit.rules()) {
+                        List<SearchImplementation.MatchPosition> matches = methodsSearchImplementation.getMatches(basicRule.getCode());
+                        if (!matches.isEmpty()) {
+                            String warningMessage = matches.stream()
+                                    .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex()))
+                                    .distinct()
+                                    .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
+                                    .collect(Collectors.joining("\n"));
+                            warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage)
+                                                        .build());
+                        }
+                    }
+                }
+            }
+
+        }
+
+        return warningMessages.stream()
+                .sorted(Comparator.comparingInt(DroolsSyntaxDeprecatedWarnings::getLine))
+                .collect(Collectors.toList());
+    }
+
 
     private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
 
+        // find deprecated elements in the ruleFileBluePrint
+        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
+        if (warningMessageForImports != null) {
+            customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports);
+        }
+        customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
+
         RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
             case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
@@ -60,35 +132,36 @@ public class DroolsSyntaxValidationService {
         if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getImportLine())
-                            .column(0)
-                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                            .build());
+                                 .line(ruleFileBluePrint.getImportLine())
+                                 .column(0)
+                                 .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
+                                 .build());
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0)
-                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                            .build());
+                                 .line(ruleFileBluePrint.getGlobalsLine())
+                                 .column(0)
+                                 .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                                 .build());
         }
-        baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
-            if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicQuery.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                .build());
-            }
-        });
+        baseRuleFileBluePrint.getQueries()
+                .forEach(basicQuery -> {
+                    if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
+                        customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                                .add(DroolsSyntaxErrorMessage.builder()
+                                             .line(basicQuery.getLine())
+                                             .column(0)
+                                             .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
+                                             .build());
+                    }
+                });
         if (ruleFileType.equals(RuleFileType.ENTITY)) {
             String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
             if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
-                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                .build());
+                customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                        .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
+                                     .build());
             }
         }
         return customSyntaxValidation;
@@ -97,7 +170,8 @@ public class DroolsSyntaxValidationService {
 
     private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+        return ruleFileBluePrint.streamAllRules()
+                .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
     }
 
 
@@ -109,13 +183,16 @@ public class DroolsSyntaxValidationService {
 
         Set<String> additionalImports = Sets.difference(imports, baseImports);
 
-        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
+        return additionalImports.stream()
+                .allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
     private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
+        return ruleFileBluePrint.getQueries()
+                .stream()
+                .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
     }
 
 
@@ -124,9 +201,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId,
-                rules.getRulesString(),
-                RuleFileType.valueOf(rules.getRuleFileType()));
+                                                                                           versionId,
+                                                                                           rules.getRulesString(),
+                                                                                           RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 
@@ -134,14 +211,18 @@ public class DroolsSyntaxValidationService {
     private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
 
         List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
+        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
+                .map(this::buildDroolsSyntaxErrorMessage)
+                .collect(Collectors.toList());
+        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages)
+                .build();
     }
 
 
     private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
 
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
+        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText())
+                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -177,8 +177,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
                                                                                               document,
                                                                                               TEST_DOSSIER_TEMPLATE_ID,
                                                                                               manualRedactions,
-                                                                                              TEST_FILE_ID,
-                                                                                              Collections.emptySet());
+                                                                                              TEST_FILE_ID);
 
         redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                       .hexColor("#ffe187")
-                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
@@ -122,12 +122,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
     }
 
-
     @Test
     public void acceptanceTests() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -136,10 +133,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished analysis");
         EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
-                .orElseThrow();
-        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
-                .orElseThrow();
+        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
@@ -151,10 +146,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
-                .orElseThrow();
-        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
-                .orElseThrow();
+        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow();
 
         assertEquals(EntryState.APPLIED, asyaLyon2.getState());
 
@@ -175,25 +168,19 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
                 .filter(entry -> entry.getValue().equals(value))
-                .filter(entry -> entry.getContainingNodeId()
-                        .get(0).equals(sectionNumber.get(0)));
+                .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
     }
 
 
     private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
 
-        return redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value));
+        return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
     }
 
 
     @Test
     public void noEndlessLoopsTest() {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -214,15 +201,13 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         assertEquals(EntryState.IGNORED,
-                     findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY))
-                             .findFirst()
-                             .get().getState());
+                findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState());
     }
 
 
     private static IdRemoval buildIdRemoval(String id) {
 
-        return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
+        return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -218,12 +218,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = ManualRedactions.builder()
                 .resizeRedactions(Set.of(ManualResizeRedaction.builder()
                                                  .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
-                                                 .fileId(TEST_FILE_ID)
                                                  .value("Dr. Alan")
                                                  .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
                                                  .requestDate(OffsetDateTime.now())
                                                  .updateDictionary(false)
-                                                 .user("user")
                                                  .build()))
                 .build();
 
@@ -585,8 +583,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void redactionTest() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         String fileName = "files/new/crafted document.pdf";
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
@@ -660,7 +656,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                              .fileId("fileId")
                                                              .type("signature")
                                                              .requestDate(OffsetDateTime.now())
-                                                             .user("user")
                                                              .build()));
 
         request.setManualRedactions(manualRedactions);
@@ -715,7 +710,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                              .annotationId("308dab9015bfafd911568cffe0a7f7de")
                              .fileId(TEST_FILE_ID)
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                             .user("user")
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
                              .build());
 
@@ -724,7 +718,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                              .annotationId("0b56ea1a87c83f351df177315af94f0d")
                              .fileId(TEST_FILE_ID)
                              .legalBasis("Something")
-                             .user("user")
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
                              .build());
@@ -733,7 +726,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .add(IdRemoval.builder()
                              .annotationId("0b56ea1a87c83f351df177315af94f0d")
                              .fileId(TEST_FILE_ID)
-                             .user("user")
                              .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
                              .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
                              .build());
@@ -953,7 +945,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .textBefore("")
                 .updateDictionary(false)
                 .textAfter("")
-                .user("user")
                 .build();
         manualRedactions.getResizeRedactions().add(manualResizeRedaction);
 
@@ -1041,14 +1032,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                                                        .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                                                        .fileId("fileId")
-                                                       .user("user")
                                                        .processedDate(OffsetDateTime.now())
                                                        .requestDate(OffsetDateTime.now())
                                                        .build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                                                            .annotationId("675eba69b0c2917de55462c817adaa05")
                                                            .fileId("fileId")
-                                                           .user("user")
                                                            .legalBasis("Something")
                                                            .requestDate(OffsetDateTime.now())
                                                            .processedDate(OffsetDateTime.now())
@@ -1238,8 +1227,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void signaturesAreRedactionAfterReanalyse() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
@@ -1261,7 +1248,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
                                                                               .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                                                                              .user("user")
                                                                               .build()))
                                             .build());
 
@@ -1281,8 +1267,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void entityIsAppliedAfterRecateorize() throws IOException {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
@@ -1304,14 +1288,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
                                                                               .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                                                                              .user("user")
                                                                               .build()))
                                             .recategorizations(Set.of(ManualRecategorization.builder()
                                                                               .annotationId("3029651d0842a625f2d23f8375c23600")
                                                                               .type("CBI_author")
+                                                                              .legalBasis("")
                                                                               .requestDate(OffsetDateTime.now())
                                                                               .fileId(TEST_FILE_ID)
-                                                                              .user("user")
                                                                               .build()))
                                             .build());
 
@@ -1455,8 +1438,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @SneakyThrows
     public void testResizeWithUpdateDictionaryTrue() {
 
-        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
         String pdfFile = "files/new/crafted document.pdf";
 
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -1474,8 +1455,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                                             .resizeRedactions(Set.of(ManualResizeRedaction.builder()
                                                                              .updateDictionary(true)
                                                                              .annotationId(david.getId())
-                                                                             .fileId(TEST_FILE_ID)
-                                                                             .user("user")
                                                                              .requestDate(OffsetDateTime.now())
                                                                              .value("David Ksenia")
                                                                              .positions(List.of(Rectangle.builder()
@@ -1503,14 +1482,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
     private IdRemoval getIdRemoval(String id) {
 
-        return IdRemoval.builder()
-                .annotationId(id)
-                .removeFromAllDossiers(false)
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .removeFromDictionary(false)
-                .requestDate(OffsetDateTime.now())
-                .build();
+        return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java

@@ -5,12 +5,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.awt.Color;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.drools.io.ClassPathResource;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -18,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@@ -47,13 +49,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -62,14 +66,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithRemovedImportsAndChangedName() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
         rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -77,102 +83,32 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithAddedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
-    @Test
-    @SneakyThrows
-    void testRulesWithUnallowedAddedImports() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertFalse(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAllRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAcceptanceRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testManualRedactionRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testRulesV2() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
     @Test
     @SneakyThrows
     void testDocumineRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -181,7 +117,8 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
@@ -196,12 +133,13 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
-                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
+                                                       "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -212,14 +150,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRemoveQuery() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", "");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -228,13 +168,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -243,13 +185,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDroolsAsEntityDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -259,13 +203,15 @@ class DroolsSyntaxValidationServiceTest {
     @Disabled
     void attemptImportsFixToAllRuleFiles() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
 
-        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
-                "drools/documine_flora.drl",
-                "drools/manual_redaction_rules.drl",
-                "drools/acceptance_rules.drl",
-                "drools/rules_v2.drl");
+        List<String> ruleFiles = List.of("drools/rules.drl",
+                                         "drools/all_redact_manager_rules.drl",
+                                         "drools/documine_flora.drl",
+                                         "drools/manual_redaction_rules.drl",
+                                         "drools/acceptance_rules.drl",
+                                         "drools/rules_v2.drl");
 
         for (String ruleFile : ruleFiles) {
             var rulesFile = new ClassPathResource(ruleFile);
@@ -288,4 +234,109 @@ class DroolsSyntaxValidationServiceTest {
 
     }
 
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAllRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAcceptanceRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testManualRedactionRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesV2() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesWithDeprecatedFunctions() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert(");
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+        assertEquals(droolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().size(), 2);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                                                                                                        .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                        .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                        .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                        .hexColor("#ffe187")
-                                                                                                        .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                        .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
@@ -155,40 +155,29 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String testEntityValue1 = "Desiree";
         String testEntityValue2 = "Melanie";
         EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertEquals(2,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue1))
-                             .count());
-        assertEquals(2,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue2))
-                             .count());
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
-        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document)
-                .findFirst()
-                .get();
+        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
         String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getValue().equals(testEntityValue1))
                 .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
-                .get().getId();
+                .get()
+                .getId();
         ManualRedactions manualRedactions = new ManualRedactions();
-        manualRedactions.getResizeRedactions()
-                .add(ManualResizeRedaction.builder()
-                             .annotationId(idToResize)
-                             .fileId(TEST_FILE_ID)
-                             .value(expandedEntityKeyword)
-                             .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
-                                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
-                             .addToAllDossiers(false)
-                             .updateDictionary(false)
-                             .requestDate(OffsetDateTime.now())
-                             .build());
+        manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
+                        .annotationId(idToResize)
+                        .value(expandedEntityKeyword)
+                        .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
+                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
+                        .addToAllDossiers(false)
+                        .updateDictionary(false)
+                        .requestDate(OffsetDateTime.now())
+                .build());
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
@@ -199,32 +188,21 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
-        EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getValue().equals(expandedEntityKeyword))
-                .findFirst()
-                .get();
+        EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
         assertEquals(idToResize, resizedEntry.getId());
+        assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
         assertEquals(1,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue1))
-                             .count());
-        assertEquals(1,
-                     redactionLog.getEntityLogEntry()
-                             .stream()
-                             .filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED))
-                             .count());
+                redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count());
     }
 
 
     private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
         return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
-                                                                                                       (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
-                                                                                                       (float) rectangle2D.getWidth(),
-                                                                                                       -(float) rectangle2D.getHeight(),
-                                                                                                       pageNumber);
+                (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 
@@ -241,10 +219,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
-                                                           .annotationId("675eba69b0c2917de55462c817adaa05")
-                                                           .fileId("fileId")
-                                                           .legalBasis("Something")
-                                                           .build()));
+                .annotationId("675eba69b0c2917de55462c817adaa05")
+                .fileId("fileId")
+                .legalBasis("Something")
+                .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
@@ -254,7 +232,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactionEntry.setValue("O'Loughlin C.K.");
         manualRedactionEntry.setReason("Manual Redaction");
         manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                                                  Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
+                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
 
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
@@ -264,11 +242,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
-                                                              .annotationId("675eba69b0c2917de55462c817adaa05")
-                                                              .fileId("fileId")
-                                                              .legalBasis("Manual Legal Basis Change")
-                                                              .requestDate(OffsetDateTime.now())
-                                                              .build())));
+                .annotationId("675eba69b0c2917de55462c817adaa05")
+                .fileId("fileId")
+                .legalBasis("Manual Legal Basis Change")
+                .requestDate(OffsetDateTime.now())
+                .build())));
 
         analyzeService.reanalyze(request);
 
@@ -317,10 +295,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
         long end = System.currentTimeMillis();
-        var optionalEntry = redactionLog.getEntityLogEntry()
-                .stream()
-                .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
-                .findAny();
+        var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
         assertTrue(optionalEntry.isPresent());
         assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
         System.out.println("duration: " + (end - start));
@@ -370,9 +345,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         assertFalse(redactionLog2.getEntityLogEntry()
-                            .stream()
-                            .filter(entry -> entry.getType().equals("published_information"))
-                            .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
+                .stream()
+                .filter(entry -> entry.getType().equals("published_information"))
+                .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
 
         var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry()
                 .stream()
@@ -406,15 +381,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String annotationId = "testAnnotationId";
 
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
-                                                        .annotationId(annotationId)
-                                                        .requestDate(OffsetDateTime.now())
-                                                        .type("manual")
-                                                        .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
-                                                        .positions(List.of(//
-                                                                           new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
-                                                                           new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
-                                                                           new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
-                                                        .build()));
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .type("manual")
+                .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
+                .positions(List.of(//
+                        new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
+                        new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
+                        new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
+                .build()));
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -32,33 +32,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualResizeRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
         Set<TextEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document)
                 .collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
-        TextEntity biggerEntity = biggerEntities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+        TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(initialId)
-                .fileId(TEST_FILE_ID)
-                .user("user")
                 .value(biggerEntity.getValue())
-                .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage()
-                                                          .get(0)))
+                .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
                 .requestDate(OffsetDateTime.now())
                 .updateDictionary(false)
                 .build();
@@ -70,13 +55,8 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
         assertEquals(biggerEntity.getPages(), entity.getPages());
         assertEquals(biggerEntity.getValue(), entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
-        assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage()
-                                            .get(0).getRectanglePerLine(),
-                                    entity.getPositionsOnPagePerPage()
-                                            .get(0).getRectanglePerLine());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
+        assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine());
         assertTrue(entity.resized());
     }
 
@@ -85,25 +65,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualForceRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -111,12 +78,8 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertFalse(entity.getIntersectingNodes().isEmpty());
         assertEquals(1, entity.getPages().size());
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals("Something",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertFalse(entity.removed());
         assertTrue(entity.hasManualChanges());
         assertTrue(entity.applied());
@@ -127,26 +90,17 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualIDRemovalTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).user("user").build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(idRemoval));
 
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertTrue(entity.ignored());
     }
 
@@ -155,25 +109,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     public void manualIDRemovalButAlsoForceRedactionTest() {
 
         Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
-                .collect(Collectors.toUnmodifiableSet());
+        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
 
-        TextEntity entity = entities.stream()
-                .filter(e -> e.getPages()
-                        .stream()
-                        .anyMatch(p -> p.getNumber() == 1))
-                .findFirst()
-                .get();
+        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
-        String initialId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .fileId(TEST_FILE_ID)
-                .user("user")
-                .build();
+        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -181,9 +123,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         assertFalse(entity.getIntersectingNodes().isEmpty());
         assertEquals(1, entity.getPages().size());
         assertEquals("David Ksenia", entity.getValue());
-        assertEquals(initialId,
-                     entity.getPositionsOnPagePerPage()
-                             .get(0).getId());
+        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
         assertFalse(entity.removed());
         assertFalse(entity.ignored());
     }
@@ -191,9 +131,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
     private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
 
-        if (rects1.stream()
-                .allMatch(rect1 -> rects2.stream()
-                        .anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
+        if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
             return;
         }
         // use this for nice formatting of error message
@@ -205,18 +143,15 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
         double tolerance = 1e-1;
         return Math.abs(r1.getX() - r2.getX()) < tolerance &&//
-               Math.abs(r1.getY() - r2.getY()) < tolerance &&//
-               Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
-               Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
+                Math.abs(r1.getY() - r2.getY()) < tolerance &&//
+                Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
+                Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
     }
 
 
     private static List<Rectangle> toAnnotationRectangles(PositionOnPage positionsOnPage) {
 
-        return positionsOnPage.getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber()))
-                .toList();
+        return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java

@@ -43,9 +43,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         OffsetDateTime start = OffsetDateTime.now();
         String reason = "whatever";
         Document document = buildGraphNoImages("files/new/crafted document.pdf");
-        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document)
-                .peek(e -> e.apply("T.0.0", reason))
-                .toList();
+        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList();
         assertFalse(entities.isEmpty());
         TextEntity entity = entities.get(0);
         assertTrue(entity.active());
@@ -54,11 +52,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.resized());
         assertFalse(entity.ignored());
         assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
-        String annotationId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).user("user").annotationId(annotationId).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -68,7 +65,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         ManualForceRedaction forceRedaction = ManualForceRedaction.builder()
                 .requestDate(start.plusSeconds(1))
                 .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
                 .legalBasis("coolio")
                 .build();
@@ -77,12 +73,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.ignored());
         assertFalse(entity.removed());
         assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
-        assertEquals("coolio",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
 
         // remove again
-        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
+        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal2);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -92,7 +86,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         ManualForceRedaction forceRedaction2 = ManualForceRedaction.builder()
                 .requestDate(start.plusSeconds(2))
                 .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
                 .legalBasis("coolio")
                 .build();
@@ -100,7 +93,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
         assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override",
-                     entity.buildReasonWithManualChangeDescriptions());
+                entity.buildReasonWithManualChangeDescriptions());
 
         String legalBasis = "Yeah";
         String section = "Some random section!";
@@ -110,7 +103,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .annotationId(annotationId)
                 .requestDate(start.plusSeconds(4))
                 .section(section)
-                .fileId(TEST_FILE_ID)
                 .user("peter")
                 .value(value)
                 .build();
@@ -118,32 +110,21 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
         assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed",
-                     entity.buildReasonWithManualChangeDescriptions());
-        assertEquals(value,
-                     entity.getManualOverwrite().getValue()
-                             .orElse(entity.getValue()));
-        assertEquals(legalBasis,
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
-        assertEquals(section,
-                     entity.getManualOverwrite().getSection()
-                             .orElse(entity.getDeepestFullyContainingNode().toString()));
+                entity.buildReasonWithManualChangeDescriptions());
+        assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue()));
+        assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
 
         ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
                 .type("type")
                 .requestDate(start.plusSeconds(5))
-                .fileId(TEST_FILE_ID)
-                .user("user")
                 .annotationId(annotationId)
+                .legalBasis("")
                 .build();
         entity.getManualOverwrite().addChange(imageRecategorizationRequest);
-        assertTrue(entity.getManualOverwrite().getRecategorized()
-                           .isPresent());
-        assertTrue(entity.getManualOverwrite().getRecategorized()
-                           .get());
-        assertEquals("type",
-                     entity.getManualOverwrite().getType()
-                             .orElse(entity.type()));
+        assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
+        assertTrue(entity.getManualOverwrite().getRecategorized().get());
+        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
     }
 
 
@@ -153,9 +134,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         OffsetDateTime start = OffsetDateTime.now();
         String reason = "whatever";
         Document document = buildGraphNoImages("files/new/crafted document.pdf");
-        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document)
-                .peek(e -> e.apply("T.0.0", reason))
-                .toList();
+        List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList();
         assertFalse(entities.isEmpty());
         TextEntity entity = entities.get(0);
         assertTrue(entity.active());
@@ -164,11 +143,10 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.resized());
         assertFalse(entity.ignored());
         assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
-        String annotationId = entity.getPositionsOnPagePerPage()
-                .get(0).getId();
+        String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -179,7 +157,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .requestDate(start.plusSeconds(1))
                 .fileId(TEST_FILE_ID)
                 .annotationId(annotationId)
-                .user("user")
                 .legalBasis("coolio")
                 .build();
         entity.getManualOverwrite().addChange(forceRedaction);
@@ -187,9 +164,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertFalse(entity.ignored());
         assertFalse(entity.removed());
         assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
-        assertEquals("coolio",
-                     entity.getManualOverwrite().getLegalBasis()
-                             .orElse(entity.getMatchedRule().getLegalBasis()));
+        assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java

@@ -84,7 +84,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
     public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
 
         DocumentAndEntity context = createFoundManualRedaction();
-        IdRemoval removal = IdRemoval.builder().annotationId("123").user("user").fileId(TEST_FILE_ID).requestDate(OffsetDateTime.now()).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
         context.entity().getManualOverwrite().addChange(removal);
         assertTrue(context.entity().removed());
     }
@@ -95,7 +95,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
     public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
 
         DocumentAndEntity context = createNotFoundManualRedaction();
-        IdRemoval removal = IdRemoval.builder().fileId(TEST_FILE_ID).user("user").annotationId("123").requestDate(OffsetDateTime.now()).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
         context.entity().getManualOverwrite().addChange(removal);
         assertTrue(context.entity().removed());
     }
@@ -108,11 +108,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         String value = "To: Syngenta Ltd. Jealott’s Hill";
         String type = DICTIONARY_AUTHOR;
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
-                .annotationId("123")
                 .type(type)
                 .value(value)
-                .user("user")
-                .fileId(TEST_FILE_ID)
                 .reason("reason")
                 .legalBasis("n-a")
                 .section("n-a")
@@ -125,20 +122,17 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
 
         assertTrue(document.getEntities().isEmpty());
 
-        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
-                                                                                                                                                .entriesToAdd(Set.of(
-                                                                                                                                                        manualRedactionEntry))
-                                                                                                                                                .build(),
-                                                                                                                                        document,
-                                                                                                                                        TEST_DOSSIER_TEMPLATE_ID);
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertEquals(1, notFoundManualEntities.size());
         assertTrue(document.getEntities().isEmpty());
 
         List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
-                                                                                                  document,
-                                                                                                  notFoundManualEntities,
-                                                                                                  new DictionaryVersion(),
-                                                                                                  0L).getEntityLogEntry();
+                document,
+                notFoundManualEntities,
+                new DictionaryVersion(),
+                0L).getEntityLogEntry();
 
         assertEquals(1, redactionLogEntries.size());
         assertEquals(value, redactionLogEntries.get(0).getValue());
@@ -152,8 +146,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         Document document = buildGraph("files/new/VV-919901.pdf");
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
 
-        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
-                .toList();
+        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
         assertFalse(tempEntities.isEmpty());
         var tempEntity = tempEntities.get(0);
         List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
@@ -165,11 +158,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
 
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
                 .type("manual")
-                .annotationId("123")
                 .value(tempEntity.getValue())
                 .reason("reason")
-                .user("user")
-                .fileId(TEST_FILE_ID)
                 .legalBasis("n-a")
                 .section(tempEntity.getDeepestFullyContainingNode().toString())
                 .rectangle(false)
@@ -182,28 +172,21 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         tempEntity.removeFromGraph();
         assertTrue(document.getEntities().isEmpty());
 
-        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
-                                                                                                                                                .entriesToAdd(Set.of(
-                                                                                                                                                        manualRedactionEntry))
-                                                                                                                                                .build(),
-                                                                                                                                        document,
-                                                                                                                                        TEST_DOSSIER_TEMPLATE_ID);
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertTrue(notFoundManualEntities.isEmpty());
         assertEquals(1, document.getEntities().size());
-        return new DocumentAndEntity(document,
-                                     document.getEntities()
-                                             .stream()
-                                             .findFirst()
-                                             .get());
+        return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
     }
 
 
     public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
         return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
-                             (float) rectangle2D.getWidth(),
-                             -(float) rectangle2D.getHeight(),
-                             pageNumber);
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.rules;
 import java.util.Collection;
 
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -11,8 +10,6 @@ import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.kie.internal.io.ResourceFactory;
-import org.mockito.Mockito;
-import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Bean;
@@ -20,13 +17,11 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
-@ExtendWith(MockitoExtension.class)
 public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
     protected static final String RULES = "drools/rules.drl";
@@ -77,12 +72,10 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
     @BeforeEach
     public void createServices() {
 
-        Dictionary dict = Mockito.mock(Dictionary.class);
         kieSession = kieContainer.newKieSession();
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
         kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
         kieSession.setGlobal("entityCreationService", entityCreationService);
-        kieSession.setGlobal("dictionary", dict);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode() {
 
-        var node = new Section(List.of(0, 1), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(0, 1), null, null, null, null,null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -29,8 +29,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode2() {
 
-        var node = new Section(Collections.emptyList(), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(Collections.emptyList(), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -42,8 +42,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode3() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode4() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null,null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -8,7 +8,6 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -35,6 +34,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
@@ -84,7 +84,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
     @SpyBean
     RabbitTemplate rabbitTemplate;
 
-
     @BeforeEach
     public void stubClients() {
 
@@ -102,22 +101,21 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
-                                                                                                       .hexColor("#ffe187")
-                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                                                                                                       .build()));
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
 
         mockDictionaryCalls(null);
 
         when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
     }
 
-
     @Test
     @SneakyThrows
     public void testManualSurroundingText() {
@@ -127,20 +125,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         ManualRedactions manualRedactions = new ManualRedactions();
 
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(355.53775f)
-                                                                                                .topLeftY(266.1895f)
-                                                                                                .width(29.32224f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "AOEL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL");
 
         var notFoundId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId,
-                                                                                 List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()),
-                                                                                 "Random");
+        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@@ -159,43 +147,30 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertFalse(unprocessedManualEntities.isEmpty());
         assertEquals(unprocessedManualEntities.size(), 2);
 
-        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream()
-                .filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId))
-                .findFirst();
+        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
         assertTrue(optionalUnprocessedManualEntity.isPresent());
 
         UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
         assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
         assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
         assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).x(), 355.53775f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).y(), 266.49002f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).w(), 29.322266f);
-        assertEquals(unprocessedManualEntity.getPositions()
-                             .get(0).h(), 11.017679f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
 
-        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream()
-                .filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId))
-                .findFirst();
+        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
         assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
 
         UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
         assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
         assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
         assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getPageNumber(), 1);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[0], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[1], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[2], 1f);
-        assertEquals(unprocessedNotFoundManualEntity.getPositions()
-                             .get(0).getRectangle()[3], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
 
         analyzeService.reanalyze(request);
 
@@ -215,14 +190,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -234,37 +205,13 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         ManualRedactions manualRedactions = new ManualRedactions();
 
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         var cormsId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId,
-                                                                                 List.of(Rectangle.builder()
-                                                                                                 .topLeftX(129.86f)
-                                                                                                 .topLeftY(505.7295f)
-                                                                                                 .width(35.9904f)
-                                                                                                 .height(10.048125f)
-                                                                                                 .page(1)
-                                                                                                 .build()),
-                                                                                 "CoRMS");
+        ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS");
 
         var a9Id = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id,
-                                                                                 List.of(Rectangle.builder()
-                                                                                                 .topLeftX(140.1096f)
-                                                                                                 .topLeftY(291.5095f)
-                                                                                                 .width(37.84512f)
-                                                                                                 .height(10.048125f)
-                                                                                                 .page(1)
-                                                                                                 .build()),
-                                                                                 "A9396G");
+        ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@@ -291,53 +238,35 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertFalse(unprocessedManualEntities.isEmpty());
         assertEquals(unprocessedManualEntities.size(), 3);
 
-        var resizedAoel = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId))
-                .findFirst();
+        var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst();
         assertTrue(resizedAoel.isPresent());
         assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
         assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
         assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(resizedAoel.get().getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight());
 
-        var cormsResized = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId))
-                .findFirst();
+        var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst();
         assertTrue(cormsResized.isPresent());
         assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
         assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
         assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).x(), positions2.get(0).getTopLeftX());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).y(), positions2.get(0).getTopLeftY());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).w(), positions2.get(0).getWidth());
-        assertEquals(cormsResized.get().getPositions()
-                             .get(0).h(), positions2.get(0).getHeight());
+        assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX());
+        assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY());
+        assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth());
+        assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight());
 
-        var a9Resized = unprocessedManualEntities.stream()
-                .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id))
-                .findFirst();
+        var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst();
         assertTrue(a9Resized.isPresent());
         assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
         assertEquals(a9Resized.get().getTextBefore(), "data for S");
         assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).x(), positions3.get(0).getTopLeftX());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).y(), positions3.get(0).getTopLeftY());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).w(), positions3.get(0).getWidth());
-        assertEquals(a9Resized.get().getPositions()
-                             .get(0).h(), positions3.get(0).getHeight());
+        assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX());
+        assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY());
+        assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth());
+        assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight());
     }
 
 
@@ -348,15 +277,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -380,14 +301,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -398,15 +315,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -430,14 +339,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
         assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
         assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).x(), positions.get(0).getTopLeftX());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).y(), positions.get(0).getTopLeftY());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).w(), positions.get(0).getWidth());
-        assertEquals(unprocessedManualEntities.get(0).getPositions()
-                             .get(0).h(), positions.get(0).getHeight());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
+        assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
     }
 
 
@@ -448,15 +353,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         ManualRedactions manualRedactions = new ManualRedactions();
         var aoelId = UUID.randomUUID().toString();
-        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
-                                                                                List.of(Rectangle.builder()
-                                                                                                .topLeftX(384.85536f)
-                                                                                                .topLeftY(240.8695f)
-                                                                                                .width(13.49088f)
-                                                                                                .height(10.048125f)
-                                                                                                .page(1)
-                                                                                                .build()),
-                                                                                "EL");
+        ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         AnalyzeRequest request = uploadFileToStorage(pdfFile);
@@ -480,32 +377,25 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
     private static ManualResizeRedaction prepareManualSizeRedaction(String id, List<Rectangle> positions, String value) {
 
-        return ManualResizeRedaction.builder()
-                .annotationId(id)
-                .fileId("fileId")
-                .user("user")
-                .positions(positions)
-                .updateDictionary(false)
-                .addToAllDossiers(false)
-                .value(value)
-                .requestDate(OffsetDateTime.now())
-                .build();
+        ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction();
+        manualResizeRedaction.setAnnotationId(id);
+        manualResizeRedaction.setPositions(positions);
+        manualResizeRedaction.setUpdateDictionary(false);
+        manualResizeRedaction.setAddToAllDossiers(false);
+        manualResizeRedaction.setValue(value);
+        return manualResizeRedaction;
     }
 
 
     private static ManualRedactionEntry prepareManualRedactionEntry(String id, List<Rectangle> positions, String value) {
 
-        return ManualRedactionEntry.builder()
-                .annotationId(id)
-                .fileId("fileId")
-                .user("user")
-                .type("CBI_author")
-                .value(value)
-                .reason("Manual Redaction")
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .positions(positions)
-                .build();
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(id);
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue(value);
+        manualRedactionEntry.setReason("Manual Redaction");
+        manualRedactionEntry.setPositions(positions);
+        return manualRedactionEntry;
     }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -16,6 +16,7 @@ public class LayoutParsingRequestProvider {
         var originFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN);
         var tablesFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES);
         var imagesFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO);
+        var sectionGridStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID);
         var structureFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE);
         var textBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT);
         var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
@@ -28,12 +29,14 @@ public class LayoutParsingRequestProvider {
                 .originFileStorageId(originFileStorageId)
                 .tablesFileStorageId(Optional.of(tablesFileStorageId))
                 .imagesFileStorageId(Optional.of(imagesFileStorageId))
+                .visualLayoutParsingFileId(Optional.empty())
                 .structureFileStorageId(structureFileStorageId)
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
                 .simplifiedTextStorageId(simplifiedTextStorageId)
                 .viewerDocumentStorageId(viewerDocumentStorageId)
+                .visualLayoutParsingFileId(Optional.empty())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -153,32 +155,6 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
 
-rule "CBI.7.2: Do not redact PII if published information found in Section without tables"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("published_information"),
-                          hasEntitiesOfType("PII"))
-    then
-        $section.getEntitiesOfType("PII")
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.2",
-                        "Published Information found in section",
-                        $section.getEntitiesOfType("published_information")
-                );
-            });
-    end
-
-rule "CBI.7.3: Do not redact PII if published information found in same table row"
-    when
-        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("PII"))
-        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
-        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $pii: TextEntity(type() == "PII", active()) from $tableCell.getEntities()
-    then
-        $pii.skipWithReferences("CBI.7.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
-    end
-
 
 // Rule unit: CBI.9
 rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
@@ -207,19 +183,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
             .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.10
 rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@@ -248,32 +211,6 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st
             .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.10.2: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "CBI.10.3: Redact all cells with Header Author as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.11
 rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
@@ -287,19 +224,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -307,12 +232,12 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -320,19 +245,7 @@ rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -357,19 +270,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 
 
 // Rule unit: CBI.20
-rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
+rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -377,12 +278,12 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
+             laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
+rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -390,117 +291,56 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\""
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.3", "PERFORMING LABORATORY was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.0: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "PII.0.2: Redact all PII (vertebrate study)"
+rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.3: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.3", "Personal Information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: PII.1
-rule "PII.1.0: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
+rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
+rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.5: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.5", "Found by Email Regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.2
-rule "PII.2.0: Redact Phone and Fax by RegEx"
+rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
@@ -513,28 +353,10 @@ rule "PII.2.0: Redact Phone and Fax by RegEx"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
+rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("Contact") ||
@@ -548,42 +370,35 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.3
-rule "PII.3.0: Redact telephone numbers by RegEx"
-    when
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
+rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
+rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.4
-rule "PII.4.0: Redact line after contact information keywords"
+rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -609,62 +424,7 @@ rule "PII.4.0: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -689,24 +449,12 @@ rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.5
-rule "PII.5.0: Redact line after contact information keywords reduced"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)"
+rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -716,10 +464,10 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)"
+rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -729,23 +477,12 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: Redact line between contact keywords"
-    when
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -754,10 +491,10 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -766,41 +503,12 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.3", "Found between contact keywords", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.7
-rule "PII.7.0: Redact contact information if applicant is found"
-    when
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -815,10 +523,10 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -833,13 +541,14 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                           containsStringIgnoreCase("producer of the active substance") ||
                           containsStringIgnoreCase("manufacturer of the active substance") ||
@@ -855,25 +564,7 @@ rule "PII.8.0: Redact contact information if producer is found"
             .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -888,17 +579,27 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
+    then
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -911,14 +612,6 @@ rule "PII.10.0: Redact study director abbreviation"
         .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
      end
 
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
-    when
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
@@ -963,66 +656,38 @@ rule "ETC.0.0: Purity Hint"
 
 
 // Rule unit: ETC.2
-rule "ETC.2.0: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
+rule "ETC.2.0: Redact signatures (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "ETC.2.2: Redact signatures (vertebrate study)"
+rule "ETC.2.1: Redact signatures (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.3: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.3", "Signature Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.3.1: Skip logos (non vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.skip("ETC.3.1", "Logo Found");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.2: Redact logos (vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.3.3: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -1037,7 +702,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
@@ -1360,6 +1024,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1414,6 +1416,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -1,957 +0,0 @@
-package drools
-
-import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.Collection;
-import java.util.stream.Stream;
-import java.util.Optional;
-
-import com.iqser.red.service.redaction.v1.server.model.document.*;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-
-global Document document
-global EntityCreationService entityCreationService
-global ManualChangesApplicationService manualChangesApplicationService
-global Dictionary dictionary
-
-//------------------------------------ queries ------------------------------------
-
-query "getFileAttributes"
-    $fileAttribute: FileAttribute()
-    end
-
-//------------------------------------ Syngenta specific rules ------------------------------------
-
-// Rule unit: SYN.1
-rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
-    when
-        $section: Section(containsString("CT") || containsString("BL"))
-    then
-        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
-        entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("SYN.1.0", ""));
-    end
-
-
-//------------------------------------ CBI rules ------------------------------------
-
-// Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
-    then
-        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
-    then
-        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: CBI.1
-rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
-    then
-        $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
-    end
-
-rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
-    then
-        $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: CBI.2
-rule "CBI.2.0: Do not redact genitive CBI Author"
-    when
-        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
-    then
-        entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
-        .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
-    end
-
-
-// Rule unit: CBI.7
-rule "CBI.7.0: Do not redact Names and Addresses if published information found in Section without tables"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("published_information"),
-                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.0",
-                        "Published Information found in section",
-                        $section.getEntitiesOfType("published_information")
-                );
-            });
-    end
-
-rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
-    when
-        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
-        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
-        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
-    then
-        $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
-    end
-
-
-// Rule unit: CBI.9
-rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: CBI.10
-rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author"))
-    then
-        $table.streamTableCellsWithHeader("Author")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: CBI.11
-rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -1
-    when
-        $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
-    then
-        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
-    end
-
-
-// Rule unit: CBI.16
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-
-// Rule unit: CBI.17
-rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
-    when
-        $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
-    then
-        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
-    end
-
-rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
-    when
-        $section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
-    then
-        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
-            .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
-    end
-
-
-// Rule unit: CBI.20
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-//------------------------------------ PII rules ------------------------------------
-
-// Rule unit: PII.0
-rule "PII.0.1: Redact all PII (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.2: Redact all PII (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: PII.1
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.3: Redact typoed Emails with indicator"
-    when
-        $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
-    then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.2
-rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("Contact") ||
-                              containsString("Telephone") ||
-                              containsString("Phone") ||
-                              containsString("Ph.") ||
-                              containsString("Fax") ||
-                              containsString("Tel") ||
-                              containsString("Ter") ||
-                              containsString("Mobile") ||
-                              containsString("Fel") ||
-                              containsString("Fer"))
-    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.2.3: Redact phone numbers without indicators"
-    when
-        $section: Section(containsString("+"))
-    then
-        entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section)
-            .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.3
-rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
-    then
-        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.7
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.8
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.10
-rule "PII.10.0: Redact study director abbreviation"
-    when
-        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
-    then
-        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
-        .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-     end
-
-
-// Rule unit: PII.11
-rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
-    when
-        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
-    then
-        entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-
-// Rule unit: PII.12
-rule "PII.12.0: Expand PII entities with salutation prefix"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
-    then
-        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
-        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.12.1: Expand PII entities with salutation prefix"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
-    then
-        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
-        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-//------------------------------------ Other rules ------------------------------------
-
-// Rule unit: ETC.0
-rule "ETC.0.0: Purity Hint"
-    when
-        $section: Section(containsStringIgnoreCase("purity"))
-    then
-        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
-            .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
-    end
-
-
-// Rule unit: ETC.2
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.2: Redact signatures (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: ETC.3
-rule "ETC.3.1: Skip logos (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.skip("ETC.3.1", "Logo Found");
-    end
-
-rule "ETC.3.2: Redact logos (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-
-// Rule unit: ETC.5
-rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
-    when
-        FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type() == "dossier_redaction")
-    then
-        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
-    when
-        not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type() == "dossier_redaction")
-    then
-        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
-        retract($dossierRedaction);
-    end
-
-
-//------------------------------------ AI rules ------------------------------------
-
-// Rule unit: AI.0
-rule "AI.0.0: Add all NER Entities of type CBI_author"
-    salience 999
-    when
-        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
-    then
-        nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
-    end
-
-
-// Rule unit: AI.1
-rule "AI.1.0: Combine and add NER Entities as CBI_address"
-    salience 999
-    when
-        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
-    then
-        entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
-    end
-
-
-//------------------------------------ Manual changes rules ------------------------------------
-
-// Rule unit: MAN.0
-rule "MAN.0.0: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $entityToBeResized: TextEntity(matchesAnnotationId($id))
-    then
-        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($entityToBeResized);
-        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.0.1: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $imageToBeResized: Image(id == $id)
-    then
-        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($imageToBeResized);
-        update($imageToBeResized.getParent());
-    end
-
-
-// Rule unit: MAN.1
-rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
-        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($entityToBeRemoved);
-        retract($idRemoval);
-        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId)
-        $imageEntityToBeRemoved: Image($id == id)
-    then
-        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($imageEntityToBeRemoved);
-        retract($idRemoval);
-        update($imageEntityToBeRemoved.getParent());
-    end
-
-
-// Rule unit: MAN.2
-rule "MAN.2.0: Apply force redaction"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId)
-        $entityToForce: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToForce.getManualOverwrite().addChange($force);
-        update($entityToForce);
-        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
-        retract($force);
-    end
-
-rule "MAN.2.1: Apply force redaction to images"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId)
-        $imageToForce: Image(id == $id)
-    then
-        $imageToForce.getManualOverwrite().addChange($force);
-        update($imageToForce);
-        update($imageToForce.getParent());
-        retract($force);
-    end
-
-
-// Rule unit: MAN.3
-rule "MAN.3.0: Apply entity recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
-    then
-        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        update($entityToBeRecategorized);
-        retract($recategorization);
-    end
-
-rule "MAN.3.1: Apply entity recategorization of same type"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
-    then
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        retract($recategorization);
-    end
-
-rule "MAN.3.2: Apply image recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
-    then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
-        retract($recategorization);
-    end
-
-rule "MAN.3.3: Apply recategorization entities by default"
-    salience 128
-    when
-        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
-    then
-        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
-    end
-
-
-// Rule unit: MAN.4
-rule "MAN.4.0: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
-        $imageToBeRecategorized: Image($id == id)
-    then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
-        update($imageToBeRecategorized)
-        retract($legalBasisChange)
-    end
-
-rule "MAN.4.1: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
-        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
-        update($entityToBeChanged)
-        retract($legalBasisChange)
-    end
-
-
-//------------------------------------ Entity merging rules ------------------------------------
-
-// Rule unit: X.0
-rule "X.0.0: Remove Entity contained by Entity of same type"
-    salience 65
-    when
-        $larger: TextEntity($type: type(), $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
-    then
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
-        retract($contained);
-    end
-
-
-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-
-// Rule unit: X.2
-rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
-    salience 64
-    when
-        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-    then
-        $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
-        retract($entity)
-    end
-
-
-// Rule unit: X.3
-rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
-    salience 64
-    when
-        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
-    salience 256
-    when
-        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
-    salience 256
-    when
-        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
-        retract($recommendation);
-    end
-
-rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
-    salience 256
-    when
-        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
-    then
-        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
-    salience 32
-    when
-        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
-    then
-        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
-        retract($lowerRank);
-    end
-
-rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
-    salience 32
-    when
-         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
-    then
-        $inner.getIntersectingNodes().forEach(node -> update(node));
-        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
-        retract($inner);
-    end
-
-
-// Rule unit: X.8
-rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
-    salience 257
-    when
-        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
-        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
-    then
-        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
-        $entity.addEngines($other.getEngines());
-        retract($other);
-    end
-
-rule "X.8.1: Remove Entity when intersected by imported Entity"
-    salience 256
-    when
-        $entity: TextEntity(engines contains Engine.IMPORTED, active())
-        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
-    then
-        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
-        retract($other);
-    end
-
-
-//------------------------------------ File attributes rules ------------------------------------
-
-// Rule unit: FA.1
-rule "FA.1.0: Remove duplicate FileAttributes"
-    salience 64
-    when
-        $fileAttribute: FileAttribute($label: label, $value: value)
-        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
-    then
-        retract($duplicate);
-    end
-
-
-//------------------------------------ Local dictionary search rules ------------------------------------
-
-// Rule unit: LDS.0
-rule "LDS.0.0: Run local dictionary search"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -999
-    when
-        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
-    then
-        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
-            .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
-                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
-            });
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -255,6 +257,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -100,18 +102,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -150,23 +151,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -193,22 +192,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -230,18 +227,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                       "CBI.8.1",
-                       "Must_redact found",
-                       "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                       $table.getEntitiesOfTypeInSameRow("must_redact", entity)
-               );
-            });
+        $authorOrAddress.applyWithReferences(
+                        "CBI.8.1",
+                        "Must_redact found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
+                );
     end
 
 
@@ -272,19 +268,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
             .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $table: Table(hasHeader("Author(s)"))
-    then
-        $table.streamTableCellsWithHeader("Author(s)")
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
 
 // Rule unit: CBI.11
 rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
@@ -298,22 +281,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
 
 
 // Rule unit: CBI.12
-rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
-        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
-        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
-        $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
-        TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
-    then
-        entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT)
-            .ifPresent(yesEntity -> {
-                yesEntity.skip("CBI.12.0", "must_redact");
-            });
-    end
-
-rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
+rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -325,13 +293,13 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
     then
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
-              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
               dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
           });
 
     end
 
-rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
+rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -344,13 +312,13 @@ rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s
 
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
-              authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
               dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
           });
 
     end
 
-rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
+rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
     when
         $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
         TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
@@ -359,19 +327,16 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
         $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
     then
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
-          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
+          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
 
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $section: Section(containsStringIgnoreCase("batches produced at"))
         $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section)
-            .forEach(entity -> entity.skip("CBI.14.0", "must_redact"));
     end
 
 
@@ -392,10 +357,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
                           containsStringIgnoreCase($residueKeyword),
                           containsStringIgnoreCase($keyword))
     then
-        entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section)
-            .forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
+        entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
+            .toList();
 
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+        $section.getEntitiesOfType(List.of($keyword, $residueKeyword))
             .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
@@ -413,8 +378,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
         $residueKeyword: String() from List.of("determination of residues", "determination of total residues")
         $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword))
     then
-        entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table)
-            .forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
+        entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
+            .toList();
 
         $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
             .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
@@ -423,19 +388,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -443,12 +396,12 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -456,19 +409,7 @@ rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.16.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.16.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -526,19 +467,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
 
 
 // Rule unit: CBI.20
-rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
+rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -546,12 +475,12 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
+             laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
+rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -559,117 +488,56 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
 
-rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\""
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
-    then
-         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
-         .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.3", "PERFORMING LABORATORY was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-             dictionary.recommendEverywhere(laboratoryEntity);
-         });
-    end
-
-
-// Rule unit: CBI.23
-rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
-    then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.0: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "PII.0.2: Redact all PII (vertebrate study)"
+rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "PII.0.3: Redact all PII"
-    when
-        $pii: TextEntity(type() == "PII", dictionaryEntry)
-    then
-        $pii.redact("PII.0.3", "Personal Information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: PII.1
-rule "PII.1.0: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
+rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
+rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.1.5: Redact Emails by RegEx"
-    when
-        $section: Section(containsString("@"))
-    then
-        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.redact("PII.1.5", "Found by Email Regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.4
-rule "PII.4.0: Redact line after contact information keywords"
+rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -695,62 +563,7 @@ rule "PII.4.0: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords"
-    when
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
-    end
-
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $contactKeyword: String() from List.of("Contact point:",
@@ -775,23 +588,12 @@ rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: Redact line between contact keywords"
-    when
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -800,10 +602,10 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -812,42 +614,12 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-
-rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-    then
-        Stream.concat(
-            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-        )
-            .forEach(contactEntity -> contactEntity.redact("PII.6.3", "Found between contact keywords", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.7
-rule "PII.7.0: Redact contact information if applicant is found"
-    when
-        $section: Section(getHeadline().containsString("applicant") ||
-                          getHeadline().containsString("Primary contact") ||
-                          getHeadline().containsString("Alternative contact") ||
-                          containsString("Applicant") ||
-                          containsString("Telephone number:"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
-    end
-
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -862,10 +634,10 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -880,13 +652,14 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                           containsStringIgnoreCase("producer of the active substance") ||
                           containsStringIgnoreCase("manufacturer of the active substance") ||
@@ -902,25 +675,7 @@ rule "PII.8.0: Redact contact information if producer is found"
             .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
-                          containsStringIgnoreCase("producer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer of the active substance") ||
-                          containsStringIgnoreCase("manufacturer:") ||
-                          containsStringIgnoreCase("Producer or producers of the active substance"))
-    then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
-            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -935,25 +690,27 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
         entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
-        .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -1000,66 +757,38 @@ rule "ETC.1.0: Redact Purity"
 
 
 // Rule unit: ETC.2
-rule "ETC.2.0: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.2.1: Redact signatures (non vertebrate study)"
+rule "ETC.2.0: Redact signatures (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "ETC.2.2: Redact signatures (vertebrate study)"
+rule "ETC.2.1: Redact signatures (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.2.3: Redact signatures"
-    when
-        $signature: Image(imageType == ImageType.SIGNATURE)
-    then
-        $signature.redact("ETC.2.3", "Signature Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "ETC.3.1: Skip logos (non vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.skip("ETC.3.1", "Logo Found");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.2: Redact logos (vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "ETC.3.3: Redact logos"
-    when
-        $logo: Image(imageType == ImageType.LOGO)
-    then
-        $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -1083,7 +812,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
@@ -1449,6 +1177,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -80,12 +82,12 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
-rule "PII.0.1: Redact all PII (non vertebrate study)"
+rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 
@@ -379,6 +381,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -500,6 +502,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -416,6 +418,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -459,7 +459,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia
     end
 
 rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
-    salience 256
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -33,7 +33,11 @@ public class RuleFileMigrator {
             rulesToAdd.forEach(ruleFileBluePrint::addRule);
         }
 
-        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
+        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
+                                                               combinedBluePrint.globals(),
+                                                               combinedBluePrint.queries(),
+                                                               ruleFileBluePrint.ruleClasses());
+        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
         String migratedFilePath = ruleFile.getAbsolutePath();
         try (var out = new FileOutputStream(migratedFilePath)) {
             out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1299,7 +1301,6 @@ rule "TAB.7.0: Indicator (Species)"
             .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
     end
 
-
 //------------------------------------ Manual redaction rules ------------------------------------
 
 // Rule unit: MAN.0
@@ -1561,6 +1562,26 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 
 //------------------------------------ File attributes rules ------------------------------------
 

redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java

@@ -25,9 +25,9 @@ public class RuleFileMigrationTest {
 
     // Put your redaction service drools paths and dossier-templates paths both RM and DM here
     static final List<String> ruleFileDirs = List.of(
-            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            "/home/kschuettler/iqser/fforesight/dossier-templates-v2",
-            "/home/kschuettler/iqser/redaction/dossier-templates-v2");
+            "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
+            "/Users/maverickstuder/Documents/DocuMine/dossier-templates-v2/",
+            "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2/");
 
 
     @Test
@@ -36,11 +36,7 @@ public class RuleFileMigrationTest {
     void migrateAllEntityRules() {
 
         for (String ruleFileDir : ruleFileDirs) {
-            Files.walk(Path.of(ruleFileDir))
-                    .filter(this::isEntityRuleFile)
-                    .map(Path::toFile)
-                    .peek(System.out::println)
-                    .forEach(RuleFileMigrator::migrateFile);
+            Files.walk(Path.of(ruleFileDir)).filter(this::isEntityRuleFile).map(Path::toFile).peek(System.out::println).forEach(RuleFileMigrator::migrateFile);
         }
     }