test publish stuff

Update .gitlab-ci.yml
disable test
2024-03-04 09:55:52 +02:00 · 2024-03-01 14:13:31 +01:00 · 2024-03-01 14:56:14 +02:00 · 2024-03-01 14:51:59 +02:00 · 2024-03-01 12:11:53 +02:00 · 2024-03-01 12:05:11 +02:00
27 changed files with 2061 additions and 91 deletions
@@ -21,3 +21,16 @@ deploy:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /^release/
    - if: $CI_COMMIT_TAG
+
+pages:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
+    - mkdir public
+    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+  artifacts:
+    paths:
+      - public
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"


-val layoutParserVersion = "0.93.0"
+val layoutParserVersion = "0.94.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.359.0"
+val persistenceServiceVersion = "2.360.0"
 val springBootStarterVersion = "3.1.5"

 configurations {
@@ -84,9 +84,6 @@ tasks.test {
    maxHeapSize = "1024m"
 }

-
-
-
 tasks.named<BootBuildImage>("bootBuildImage") {

    environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -113,3 +110,40 @@ tasks.named<BootBuildImage>("bootBuildImage") {
        tags.set(listOf(dockerTag))
    }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}
@@ -11,7 +11,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;

 public class MigrationMapper {

@@ -46,12 +45,12 @@ public class MigrationMapper {
    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {

        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
@@ -207,8 +207,7 @@ public final class MigrationEntity {
        return entityLogEntry.getManualChanges()
                .stream()
                .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType()
-                        .equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY))
+                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
                .isPresent();
    }

@@ -35,6 +35,7 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
+
    @Builder.Default
    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));

@@ -73,7 +74,9 @@ public class Image implements GenericSemanticNode, IEntity {
    @Override
    public TextBlock getTextBlock() {

-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
    }


@@ -94,7 +97,8 @@ public class Image implements GenericSemanticNode, IEntity {
    @Override
    public String type() {

-        return getManualOverwrite().getType().orElse(imageType.toString());
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString());
    }


@@ -126,15 +130,33 @@ public class Image implements GenericSemanticNode, IEntity {
        return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
    }

-    public boolean mostlyContainedBy(Image image) {
-        Map<Page,Rectangle2D> bboxImage = image.getBBox();
-        Map<Page,Rectangle2D> bbox = this.getBBox();
-        Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if(bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
-        return (calculatedIntersection / area) > 0.8;
+        return (area / calculatedIntersection) > containmentThreshold;
    }

+
    public int length() {

        return 0;
@@ -26,10 +26,9 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
-@FieldDefaults(makeFinal=true, level= AccessLevel.PRIVATE)
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityChangeLogService {

-
    @Timed("redactmanager_computeChanges")
    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) {

@@ -42,7 +41,9 @@ public class EntityChangeLogService {
        boolean hasChanges = false;

        for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
+            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
+                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
+                    .findAny();
            if (optionalPreviousEntity.isEmpty()) {
                hasChanges = true;
                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -63,15 +64,22 @@ public class EntityChangeLogService {


    private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
-                                                   List<EntityLogEntry> newEntityLogEntries,
-                                                   ManualRedactions manualRedactions,
-                                                   int analysisNumber,
-                                                   OffsetDateTime now) {
+                                            List<EntityLogEntry> newEntityLogEntries,
+                                            ManualRedactions manualRedactions,
+                                            int analysisNumber,
+                                            OffsetDateTime now) {

-        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        Set<String> existingIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                .filter(entry -> !existingIds.contains(entry.getId()))
+                .collect(Collectors.toList());
+        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
+                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
                .toList();
+        previousEntityLogEntries.removeAll(removedDossierRedaction);
+        removedEntries.removeAll(removedDossierRedaction);
        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
        removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
@@ -85,13 +93,15 @@ public class EntityChangeLogService {
            return;
        }

-        manualRedactions.getIdsToRemove().stream()
+        manualRedactions.getIdsToRemove()
+                .stream()
                .filter(IdRemoval::isRemoveFromDictionary)//
                .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
                .findFirst()//
                .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
    }

+
    private ChangeType calculateChangeType(EntryState state, EntryState previousState) {

        if (state.equals(previousState)) {
@@ -125,7 +125,7 @@ public class EntityLogCreatorService {
                .stream()
                .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
                                                                                                                                                           .get(0))))
-                .toList();
+                .collect(Collectors.toList());
        previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);

        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
@@ -33,14 +33,14 @@ public class ManualChangeFactory {
        if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
        } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
-            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY);
-        } else if (baseAnnotation instanceof ManualForceRedaction) {
-            manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
+        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
+            manualChange.withManualRedactionType(ManualRedactionType.FORCE).withChange("legalBasis", manualForceRedaction.getLegalBasis());
        } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
                    .withChange("value", manualResizeRedact.getValue());
        } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
-            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
+            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD)
                    .withChange("value", manualRedactionEntry.getValue());
        } else if (baseAnnotation instanceof ManualLegalBasisChange manualLegalBasisChange) {
            manualChange.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
@@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
@@ -25,15 +26,12 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

 import io.micrometer.observation.annotation.Observed;
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -51,20 +49,10 @@ public class UnprocessedChangesService {
    final ObservedStorageService observedStorageService;
    final EntityFindingUtility entityFindingUtility;
    final RedactionStorageService redactionStorageService;
-    final EntityEnrichmentService entityEnrichmentService;
    final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
    final DictionaryService dictionaryService;
    final ManualChangesApplicationService manualChangesApplicationService;

-    EntityCreationService entityCreationService;
-
-
-    @PostConstruct
-    public void initEntityCreationService() {
-
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-    }
-

    @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
    public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
@@ -201,6 +189,7 @@ public class UnprocessedChangesService {
        return manualRedactions.getEntriesToAdd()
                .stream()
                .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
+                .filter(BaseAnnotation::isLocal)
                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
                                                                                      dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                .toList();
@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -50,12 +51,12 @@ public class EntityFromPrecursorCreationService {
        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
        List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
                .stream()
+                .filter(BaseAnnotation::isLocal)
                .filter(manualRedactionEntry -> idRemovals.stream()
                        .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
                        .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
                        .findAny()//
                        .isEmpty())
-                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
                .map(manualRedactionEntry -> //
                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                .peek(manualEntity -> {
@@ -66,7 +67,6 @@ public class EntityFromPrecursorCreationService {
                    }
                })
                .toList();
-
        return toTextEntity(manualEntities, node);
    }

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -55,7 +56,14 @@ public class EntityDroolsExecutionService {
                                            ManualRedactions manualRedactions,
                                            NerEntities nerEntities) {

-        return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
+        return executeRules(kieContainer,
+                            document,
+                            document.streamChildren()
+                                    .toList(),
+                            dictionary,
+                            fileAttributes,
+                            manualRedactions,
+                            nerEntities);
    }


@@ -80,19 +88,28 @@ public class EntityDroolsExecutionService {
        kieSession.setGlobal("dictionary", dictionary);

        kieSession.insert(document);
-        document.getEntities().forEach(kieSession::insert);
+
+        document.getEntities()
+                .forEach(kieSession::insert);
+
        sectionsToAnalyze.forEach(kieSession::insert);
-        sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
-        document.getPages().forEach(kieSession::insert);
-        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
+
+        sectionsToAnalyze.stream()
+                .flatMap(SemanticNode::streamAllSubNodes)
+                .forEach(kieSession::insert);
+
+        document.getPages()
+                .forEach(kieSession::insert);
+
+        fileAttributes.stream()
+                .filter(f -> f.getValue() != null)
+                .forEach(kieSession::insert);

        if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
-            manualRedactions.getRecategorizations().forEach(kieSession::insert);
-            manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
-            manualRedactions.getForceRedactions().forEach(kieSession::insert);
-            manualRedactions.getIdsToRemove().forEach(kieSession::insert);
-            manualRedactions.getLegalBasisChanges().forEach(kieSession::insert);
+            manualRedactions.buildAll()
+                    .stream()
+                    .filter(BaseAnnotation::isLocal)
+                    .forEach(kieSession::insert);
        }

        kieSession.insert(nerEntities);
@@ -105,7 +122,8 @@ public class EntityDroolsExecutionService {
        });

        try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
+                    .get();
        } catch (ExecutionException e) {
            kieSession.dispose();
            if (e.getCause() instanceof TimeoutException) {
@@ -288,7 +288,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
        }
        if (entityLogEntry.getManualChanges()
                .stream()
-                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || mc.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT))) {
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE))) {
            assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
        }
        assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
@@ -1475,7 +1475,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                .findFirst()
                .get();
        assertEquals(resizedEntity.getState(), EntryState.APPLIED);
-        assertEquals(resizedEntity.getValue(), "David Ksenia");
+        assertEquals(resizedEntity.getValue(), "David");
+        assertEquals(0, resizedEntity.getManualChanges().size());
    }


@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -690,13 +692,22 @@ rule "ETC.3.1: Redact logos (vertebrate study)"


 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
    end


@@ -846,7 +857,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1013,6 +1024,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1148,13 +1150,22 @@ rule "ETC.4.2: Redact dossier dictionary entries"


 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
    end


@@ -1409,7 +1420,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1576,6 +1587,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1276,7 +1278,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1414,6 +1416,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -188,7 +190,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -255,6 +257,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------

 // Rule unit: LDS.0
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -800,13 +802,22 @@ rule "ETC.4.0: Redact dossier dictionary entries"


 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
    end


@@ -999,7 +1010,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1166,6 +1177,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -212,7 +214,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -379,6 +381,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -338,7 +340,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -500,6 +502,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -238,7 +240,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -416,6 +418,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
    end


+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -449,14 +449,22 @@ rule "ETC.3.1: Redact logos (non vertebrate study)"


 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        update($dossierRedaction);
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
    end


@@ -33,7 +33,11 @@ public class RuleFileMigrator {
            rulesToAdd.forEach(ruleFileBluePrint::addRule);
        }

-        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
+        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
+                                                               combinedBluePrint.globals(),
+                                                               combinedBluePrint.queries(),
+                                                               ruleFileBluePrint.ruleClasses());
+        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
        String migratedFilePath = ruleFile.getAbsolutePath();
        try (var out = new FileOutputStream(migratedFilePath)) {
            out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));
@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1157,13 +1159,22 @@ rule "ETC.4.2: Redact dossier dictionary entries"


 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
    end


@@ -1432,7 +1443,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1597,6 +1608,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
        retract($other);
    end

+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end

 //------------------------------------ File attributes rules ------------------------------------

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1299,7 +1301,6 @@ rule "TAB.7.0: Indicator (Species)"
            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
    end

-
 //------------------------------------ Manual redaction rules ------------------------------------

 // Rule unit: MAN.0
@@ -1423,7 +1424,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end
@@ -1561,6 +1562,26 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
        retract($other);
    end

+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+

 //------------------------------------ File attributes rules ------------------------------------
Author	SHA1	Message	Date
Andrei Isvoran	97291ffcae	test publish stuff	2024-03-04 09:55:52 +02:00
Kevin Tumma	9a9867e739	Update .gitlab-ci.yml	2024-03-01 14:13:31 +01:00
Andrei Isvoran	aa20a5c201	disable test	2024-03-01 14:56:14 +02:00
Andrei Isvoran	600394c366	change gitlab-ci	2024-03-01 14:51:59 +02:00
Andrei Isvoran	8fd9855b2c	add public to it	2024-03-01 12:11:53 +02:00
Andrei Isvoran	2a8f1ecdd2	add public to it	2024-03-01 12:05:11 +02:00
Andrei Isvoran	6202e95ad8	change to pages	2024-03-01 11:59:26 +02:00
Andrei Isvoran	b1fee50718	change to pages	2024-03-01 11:53:27 +02:00
Andrei Isvoran	3a59bff246	add publish keyword	2024-03-01 11:49:19 +02:00
Andrei Isvoran	57683d49b6	fix path	2024-03-01 11:48:06 +02:00
Andrei Isvoran	0fd01fa1c3	add dependsOn(delombok)	2024-03-01 11:44:37 +02:00
Andrei Isvoran	6d66e5dac3	add sleep	2024-03-01 11:24:59 +02:00
Andrei Isvoran	446c3e56a8	draft	2024-03-01 11:23:45 +02:00
Andrei Isvoran	daab1f1b0a	draft	2024-03-01 10:10:24 +02:00
Andrei Isvoran	af70705250	draft	2024-03-01 10:06:55 +02:00
Andrei Isvoran	f29f8626e9	draft	2024-03-01 09:59:31 +02:00
Andrei Isvoran	940e6e7b42	draft	2024-03-01 09:52:13 +02:00
Andrei Isvoran	c08ad75893	draft	2024-03-01 09:49:55 +02:00
Andrei Isvoran	b8d9b101c9	draft	2024-03-01 09:47:28 +02:00
Andrei Isvoran	a1ea354225	draft	2024-03-01 09:39:53 +02:00
Andrei Isvoran	e9f516580e	draft	2024-02-29 17:48:02 +02:00
Andrei Isvoran	b006b8c1ab	draft	2024-02-29 17:36:31 +02:00
Andrei Isvoran	78912adeb6	draft	2024-02-29 17:33:23 +02:00
Andrei Isvoran	9bbff158c0	draft	2024-02-29 17:28:33 +02:00
Andrei Isvoran	6e0b343de3	draft	2024-02-29 17:21:39 +02:00
Andrei Isvoran	5bc9b60623	draft	2024-02-29 17:16:09 +02:00
Andrei Isvoran	e8b32681ce	draft	2024-02-29 16:40:59 +02:00
Andrei Isvoran	d29b6bf816	draft	2024-02-29 16:39:09 +02:00
Corina Olariu	b6ad1b5e91	Merge branch 'RED-8590' into 'master' RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature Closes RED-8590 See merge request redactmanager/redaction-service!293	2024-02-29 13:17:56 +01:00
Corina Olariu	5f961b0f99	RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature	2024-02-29 13:17:56 +01:00
Andrei Isvoran	89dea4138a	Merge branch 'RED-8586' into 'master' RED-8586 - Fix confidentiality rules Closes RED-8586 See merge request redactmanager/redaction-service!294	2024-02-29 12:58:23 +01:00
Andrei Isvoran	4fb8b10429	RED-8586 - Fix confidentiality rules	2024-02-29 12:58:23 +01:00
Yannik Hampe	f09a4e4d42	Merge branch 'RED-8481-rules' into 'master' RED-8481: use visual layout parser for signature extraction Closes RED-8481 See merge request redactmanager/redaction-service!296	2024-02-29 09:19:41 +01:00
Yannik Hampe	ab5dc2f46b	RED-8481: use visual layout parser for signature extraction	2024-02-29 09:19:41 +01:00
Kilian Schüttler	b50899e18c	Merge branch 'RED-8615' into 'master' RED-8615: Revert and change the current behaviour of manualChanges Closes RED-8615 See merge request redactmanager/redaction-service!291	2024-02-28 13:12:40 +01:00
Kilian Schüttler	30d0fd7def	RED-8615: Revert and change the current behaviour of manualChanges	2024-02-28 13:12:40 +01:00
Yannik Hampe	23b8d164d2	Merge branch 'RED-8481-rules' into 'master' RED-8481: use visual layout parser for signature extraction Closes RED-8481 See merge request redactmanager/redaction-service!290	2024-02-28 09:04:55 +01:00
Yannik Hampe	0bfd92846f	RED-8481: use visual layout parser for signature extraction	2024-02-28 09:04:55 +01:00