Merge branch 'RED-7384-fp' into 'master'

RED-7384: handle pending dict application in redaction-service instead of persistence Closes RED-7384 See merge request redactmanager/redaction-service!357
RED-7384: handle pending dict application in redaction-service instead of persistence
2024-04-03 16:32:08 +02:00 · 2024-04-03 16:32:08 +02:00 · 2024-04-03 15:33:10 +02:00 · 2024-04-03 15:33:10 +02:00 · 2024-04-03 15:18:06 +02:00 · 2024-04-03 15:18:06 +02:00
37 changed files with 826 additions and 263 deletions
@@ -7,18 +7,13 @@ include:
    ref: 'main'
    file: 'ci-templates/gradle_java.yml'

-deploy:
+deploy JavaDoc:
  stage: deploy
  tags:
    - dind
  script:
-    - echo "Building with gradle version ${BUILDVERSION}"
+    - echo "Building JavaDoc with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
-    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
-  artifacts:
-    reports:
-      dotenv: version.env
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /^release/
@@ -37,4 +32,4 @@ generateJavaDoc:
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /^release/
-    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_TAG
@@ -1,11 +1,15 @@
 package com.iqser.red.service.redaction.v1.model;

+import java.util.Collections;
+import java.util.Set;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.NonNull;

@Data
@Builder
@@ -13,9 +17,18 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class MigrationRequest {

+    @NonNull
    String dossierTemplateId;
+    @NonNull
    String dossierId;
+    @NonNull
    String fileId;
+
+    boolean fileIsApproved;
+    @NonNull
    ManualRedactions manualRedactions;
+    @NonNull
+    @Builder.Default
+    Set<String> entitiesWithComments = Collections.emptySet();

 }
@@ -16,7 +16,7 @@ val layoutParserVersion = "0.96.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.377.0"
+val persistenceServiceVersion = "2.380.0"
 val springBootStarterVersion = "3.1.5"

 configurations {
@@ -206,7 +206,8 @@ public class LegacyRedactionLogMergeService {
                             .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
                             .withChange("type", imageRecategorization.getType())
                             .withChange("section", imageRecategorization.getSection())
-                             .withChange("legalBasis", imageRecategorization.getLegalBasis()));
+                             .withChange("legalBasis", imageRecategorization.getLegalBasis())
+                             .withChange("value", imageRecategorization.getValue()));
    }


@@ -57,7 +57,7 @@ public class MigrationMessageReceiver {

        if (redactionLog.getAnalysisVersion() == 0) {
            redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else if (migrationRequest.getManualRedactions() != null) {
+        } else {
            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
                                                                                                      migrationRequest.getManualRedactions(),
                                                                                                      migrationRequest.getDossierTemplateId());
@@ -67,7 +67,9 @@ public class MigrationMessageReceiver {
                                                                                              document,
                                                                                              migrationRequest.getDossierTemplateId(),
                                                                                              migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId());
+                                                                                              migrationRequest.getFileId(),
+                                                                                              migrationRequest.getEntitiesWithComments(),
+                                                                                              migrationRequest.isFileIsApproved());

        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
@@ -8,6 +8,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -19,7 +20,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@@ -57,7 +60,13 @@ public class RedactionLogToEntityLogMigrationService {
    ManualChangesApplicationService manualChangesApplicationService;


-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog,
+                                     Document document,
+                                     String dossierTemplateId,
+                                     ManualRedactions manualRedactions,
+                                     String fileId,
+                                     Set<String> entitiesWithComments,
+                                     boolean fileIsApproved) {

        log.info("Migrating entities for file {}", fileId);
        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
@@ -66,7 +75,7 @@ public class RedactionLogToEntityLogMigrationService {
                .collect(new MigratedIdsCollector());

        log.info("applying manual changes to migrated entities for file {}", fileId);
-        applyManualChanges(entitiesToMigrate, manualRedactions);
+        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);

        EntityLog entityLog = new EntityLog();
        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -87,7 +96,7 @@ public class RedactionLogToEntityLogMigrationService {
                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
                                            .toList());

-        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
+        if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
                                           redactionLog.getRedactionLogEntry().size(),
                                           entityLog.getEntityLogEntry().size());
@@ -95,8 +104,14 @@ public class RedactionLogToEntityLogMigrationService {
            throw new AssertionError(message);
        }

+        Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
+                .stream()
+                .filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
+                .map(BaseAnnotation::getAnnotationId)
+                .collect(Collectors.toSet());
+
        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(MigrationEntity::hasManualChangesOrComments)
+                .filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
                .filter(m -> !m.getOldId().equals(m.getNewId()))
                .collect(new MigratedIdsCollector());

@@ -111,20 +126,29 @@ public class RedactionLogToEntityLogMigrationService {
    }


-    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
+    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {

        if (manualRedactions == null) {
            return;
        }
+        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;

-        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
-                                                                                   manualRedactions.getEntriesToAdd(),
-                                                                                   manualRedactions.getForceRedactions(),
-                                                                                   manualRedactions.getResizeRedactions(),
-                                                                                   manualRedactions.getLegalBasisChanges(),
-                                                                                   manualRedactions.getRecategorizations())
-                .flatMap(Collection::stream)
-                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+        if (fileIsApproved) {
+            manualChangesPerAnnotationId = manualRedactions.buildAll()
+                    .stream()
+                    .filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
+                                            // unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
+                                            || (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
+                                                && (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
+                    .map(this::convertPendingDictChangesToLocal)
+                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+        } else {
+            manualChangesPerAnnotationId = manualRedactions.buildAll()
+                    .stream()
+                    .filter(manualChange -> manualChange.getProcessedDate() != null)
+                    .filter(BaseAnnotation::isLocal)
+                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+        }

        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
                                                                                                                                  Collections.emptyList()),
@@ -133,15 +157,40 @@ public class RedactionLogToEntityLogMigrationService {
    }


-    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
+    private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {

-        return redactionLog.getRedactionLogEntry().size();
+        if (baseAnnotation.getProcessedDate() != null) {
+            return baseAnnotation;
+        }
+
+        if (baseAnnotation.isLocal()) {
+            return baseAnnotation;
+        }
+
+        if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
+            manualResizeRedaction.setAddToAllDossiers(false);
+            manualResizeRedaction.setUpdateDictionary(false);
+        } else if (baseAnnotation instanceof IdRemoval idRemoval) {
+            idRemoval.setRemoveFromAllDossiers(false);
+            idRemoval.setRemoveFromDictionary(false);
+        }
+
+        return baseAnnotation;
+    }
+
+
+    private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
+
+        return redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
+                .count();
    }


    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {

-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
        return Stream.of(textMigrationEntities.stream(), images.stream())
                .flatMap(Function.identity())
@@ -155,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
    }


-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {

        List<Image> images = document.streamAllImages()
                .collect(Collectors.toList());
@@ -202,7 +251,7 @@ public class RedactionLogToEntityLogMigrationService {
            } else {
                closestImage.skip(ruleIdentifier, reason);
            }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
        }
        return migrationEntities;
    }
@@ -248,7 +297,8 @@ public class RedactionLogToEntityLogMigrationService {
        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                .stream()
                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
+                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
                .toList();

        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
@@ -285,4 +335,20 @@ public class RedactionLogToEntityLogMigrationService {
        return entitiesToMigrate;
    }

+
+    private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
+
+        var pages = redactionLogEntry.getPositions()
+                .stream()
+                .map(Rectangle::getPage)
+                .collect(Collectors.toSet());
+
+        for (int page : pages) {
+            if (page > numberOfPages) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }
@@ -8,6 +8,7 @@ import java.time.OffsetDateTime;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -19,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;

@@ -48,6 +52,8 @@ public final class MigrationEntity {

    private final PrecursorEntity precursorEntity;
    private final RedactionLogEntry redactionLogEntry;
+    private final DictionaryService dictionaryService;
+    private final String dossierTemplateId;
    private IEntity migratedEntity;
    private String oldId;
    private String newId;
@@ -57,10 +63,10 @@ public final class MigrationEntity {
    List<BaseAnnotation> manualChanges = new LinkedList<>();


-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {

+        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
-
        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
        } else if (redactionLogEntry.lastChangeIsRemoved()) {
@@ -75,13 +81,32 @@ public final class MigrationEntity {
            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
        }

-        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
+        return MigrationEntity.builder()
+                .precursorEntity(precursorEntity)
+                .redactionLogEntry(redactionLogEntry)
+                .oldId(redactionLogEntry.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
    }


-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
+                                                        Image image,
+                                                        String fileId,
+                                                        DictionaryService dictionaryService,
+                                                        String dossierTemplateId) {

-        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
+        return MigrationEntity.builder()
+                .redactionLogEntry(redactionLogImage)
+                .migratedEntity(image)
+                .oldId(redactionLogImage.getId())
+                .newId(image.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
    }


@@ -158,18 +183,6 @@ public final class MigrationEntity {
    }


-    private static EntryType getEntryType(EntityType entityType) {
-
-        return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
-            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
-            case RECOMMENDATION -> EntryType.RECOMMENDATION;
-            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
-        };
-    }
-
-
    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {

        EntityLogEntry entityLogEntry;
@@ -192,9 +205,6 @@ public final class MigrationEntity {
        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
-        if (redactionLogEntry.getLegalBasis() != null) {
-            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
-        }

        if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
            entityLogEntry.setState(EntryState.IGNORED);
@@ -227,13 +237,15 @@ public final class MigrationEntity {

    public EntityLogEntry createEntityLogEntry(Image image) {

+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
        return EntityLogEntry.builder()
                .id(image.getId())
-                .value(image.value())
-                .type(image.type())
+                .value(image.getValue())
+                .type(imageType)
                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
+                .legalBasis(image.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
                .dictionaryEntry(false)
                .positions(positions)
@@ -245,7 +257,7 @@ public final class MigrationEntity {
                .textBefore(redactionLogEntry.getTextBefore())
                .imageHasTransparency(image.isTransparent())
                .state(buildEntryState(image))
-                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                .build();

    }
@@ -256,7 +268,8 @@ public final class MigrationEntity {
        return EntityLogEntry.builder()
                .id(precursorEntity.getId())
                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.legalBasis())
+                .legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
                .value(precursorEntity.value())
                .type(precursorEntity.type())
                .state(buildEntryState(precursorEntity))
@@ -290,7 +303,8 @@ public final class MigrationEntity {
                .id(entity.getId())
                .positions(rectanglesPerLine)
                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
+                .legalBasis(entity.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
                .value(entity.getManualOverwrite().getValue()
                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                .type(entity.type())
@@ -333,11 +347,11 @@ public final class MigrationEntity {
    }


-    public boolean hasManualChangesOrComments() {
+    public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {

        return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges();
+               || hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
    }


@@ -354,6 +368,9 @@ public final class MigrationEntity {
            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
                manualResizeRedaction.setAnnotationId(newId);
                manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
+            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
+                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
+                migratedEntity.getManualOverwrite().addChange(manualChange);
            } else {
                migratedEntity.getManualOverwrite().addChange(manualChange);
            }
@@ -371,10 +388,7 @@ public final class MigrationEntity {
                .findFirst()
                .orElse(manualChanges.get(0)).getUser();

-        OffsetDateTime requestDate = manualChanges.stream()
-                .filter(mc -> mc instanceof ManualResizeRedaction)
-                .findFirst()
-                .orElse(manualChanges.get(0)).getRequestDate();
+        OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();

        return ManualRedactionEntry.builder()
                .annotationId(newId)
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;

 import java.io.Serializable;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -11,11 +10,11 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;

-import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;

@Data
-@AllArgsConstructor
+@Slf4j
 public class DictionaryModel implements Serializable {

    private final String type;
@@ -29,6 +28,7 @@ public class DictionaryModel implements Serializable {
    private final Set<DictionaryEntryModel> falseRecommendations;

    private transient SearchImplementation entriesSearch;
+    private transient SearchImplementation deletionEntriesSearch;
    private transient SearchImplementation falsePositiveSearch;
    private transient SearchImplementation falseRecommendationsSearch;

@@ -56,20 +56,6 @@ public class DictionaryModel implements Serializable {
        this.entries = entries;
        this.falsePositives = falsePositives;
        this.falseRecommendations = falseRecommendations;
-
-        this.entriesSearch = new SearchImplementation(this.entries.stream()
-                                                              .filter(e -> !e.isDeleted())
-                                                              .map(DictionaryEntryModel::getValue)
-                                                              .collect(Collectors.toList()), caseInsensitive);
-        this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
-                                                                    .filter(e -> !e.isDeleted())
-                                                                    .map(DictionaryEntryModel::getValue)
-                                                                    .collect(Collectors.toList()), caseInsensitive);
-        this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
-                                                                           .filter(e -> !e.isDeleted())
-                                                                           .map(DictionaryEntry::getValue)
-                                                                           .collect(Collectors.toList()), caseInsensitive);
-
    }


@@ -94,6 +80,18 @@ public class DictionaryModel implements Serializable {
    }


+    public SearchImplementation getDeletionEntriesSearch() {
+
+        if (deletionEntriesSearch == null) {
+            this.deletionEntriesSearch = new SearchImplementation(this.entries.stream()
+                                                                          .filter(DictionaryEntry::isDeleted)
+                                                                          .map(DictionaryEntry::getValue)
+                                                                          .collect(Collectors.toList()), caseInsensitive);
+        }
+        return deletionEntriesSearch;
+    }
+
+
    public SearchImplementation getFalsePositiveSearch() {

        if (falsePositiveSearch == null) {
@@ -5,5 +5,6 @@ public enum EntityType {
    HINT,
    RECOMMENDATION,
    FALSE_POSITIVE,
-    FALSE_RECOMMENDATION
+    FALSE_RECOMMENDATION,
+    DICTIONARY_REMOVAL
 }
@@ -132,6 +132,7 @@ public class ManualChangeOverwrite {
                recategorized = true;
                type = recategorization.getType();
                section = recategorization.getSection();
+                value = recategorization.getValue();
                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
                    legalBasis = recategorization.getLegalBasis();
                }
@@ -25,7 +25,8 @@ public final class MatchedRule implements Comparable<MatchedRule> {
    public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
    public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
    public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
+    public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);

    RuleIdentifier ruleIdentifier;
    @Builder.Default
@@ -40,7 +40,7 @@ public class TextEntity implements IEntity {
    TextRange textRange;
    @Builder.Default
    List<TextRange> duplicateTextRanges = new ArrayList<>();
-    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
+    String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
    final EntityType entityType;

    @Builder.Default
@@ -182,25 +182,37 @@ public class TextEntity implements IEntity {

    public boolean containedBy(TextEntity textEntity) {

-        return this.textRange.containedBy(textEntity.getTextRange())
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange))
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::containedBy));
+        return this.textRange.containedBy(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::containedBy));
    }


    public boolean contains(TextEntity textEntity) {

-        return this.textRange.contains(textEntity.getTextRange())
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange))
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::contains));
+        return this.textRange.contains(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::contains));
    }


    public boolean intersects(TextEntity textEntity) {

-        return this.textRange.intersects(textEntity.getTextRange())
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange))
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::intersects));
+        return this.textRange.intersects(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::intersects));
    }


@@ -6,10 +6,11 @@ import java.util.Set;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;

@@ -38,10 +39,13 @@ public class DictionarySearchService {
    @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
    public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {

-        for (var model : dictionary.getDictionaryModels()) {
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
            bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
            bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
+            if (model.isDossierDictionary()) {
+                bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+            }
        }
    }

@@ -52,11 +56,12 @@ public class DictionarySearchService {
                                                   SemanticNode node,
                                                   boolean isDossierDictionaryEntry) {

+        Set<Engine> engines = isDossierDictionaryEntry ? Set.of(Engine.DICTIONARY, Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
        searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                .stream()
                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY))
+                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
                        .ifPresent(entity -> {
                            entity.setDictionaryEntry(true);
                            entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
@@ -4,6 +4,7 @@ import java.awt.Color;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
@@ -91,7 +92,8 @@ public class DictionaryService {
            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
        }

-        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
+        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion)
+                .build();
    }


@@ -368,30 +370,32 @@ public class DictionaryService {
    @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
    public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {

-        List<DictionaryModel> mergedDictionaries;
+        List<DictionaryModel> mergedDictionaries = new LinkedList<>();

-        var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
-        var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
+        List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));

-        // merge dictionaries if they have same names
+        // add dossier
        long dossierDictionaryVersion = -1;
        if (dossierDictionaryExists(dossierId)) {
-            var dossierRepresentation = getDossierDictionary(dossierId);
-            var dossierDictionaries = dossierRepresentation.getDictionary();
-            mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
-                                                                                                          convertDictionaryModel(dossierDictionaries)));
-            dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
+            DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
+            List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
+            dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+            return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion());
        } else {
-            mergedDictionaries = new ArrayList<>();
-            dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+            return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion);
        }

+    }
+
+
+    private Dictionary getDictionary(List<DictionaryModel> mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) {
+
        return new Dictionary(mergedDictionaries.stream()
                                      .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                                      .collect(Collectors.toList()),
-                              DictionaryVersion.builder()
-                                      .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
-                                      .dossierVersion(dossierDictionaryVersion)
+                              DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion)
                                      .build());
    }

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service;

 import java.time.OffsetDateTime;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -13,9 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;

 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -41,9 +37,11 @@ public class EntityChangeLogService {
        boolean hasChanges = false;

        for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
+
            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                    .findAny();
+
            if (optionalPreviousEntity.isEmpty()) {
                hasChanges = true;
                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -51,14 +49,18 @@ public class EntityChangeLogService {
            }

            EntityLogEntry previousEntity = optionalPreviousEntity.get();
+
            entityLogEntry.getChanges().addAll(previousEntity.getChanges());
+
            if (!previousEntity.getState().equals(entityLogEntry.getState())) {
                hasChanges = true;
                ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
                entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
            }
        }
+
        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
+
        return hasChanges;
    }

@@ -68,11 +70,16 @@ public class EntityChangeLogService {
        Set<String> existingIds = newEntityLogEntries.stream()
                .map(EntityLogEntry::getId)
                .collect(Collectors.toSet());
+
        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                .filter(entry -> !existingIds.contains(entry.getId()))
                .toList();
-        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
-        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+
+        removedEntries.stream()
+                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
+                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
+                .forEach(entry -> entry.setState(EntryState.REMOVED));
+
        newEntityLogEntries.addAll(removedEntries);
    }

@@ -52,9 +52,11 @@ public class EntityLogCreatorService {
    RedactionStorageService redactionStorageService;


-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
+    private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {

-        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
+        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) //
+                 || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION) //
+                 || textEntity.getEntityType().equals(EntityType.DICTIONARY_REMOVAL));
    }


@@ -145,7 +147,7 @@ public class EntityLogCreatorService {
        document.getEntities()
                .stream()
                .filter(entity -> !entity.getValue().isEmpty())
-                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                .filter(entity -> !entity.removed())
                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
        document.streamAllImages()
@@ -188,7 +190,7 @@ public class EntityLogCreatorService {
        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
        return EntityLogEntry.builder()
                .id(image.getId())
-                .value(image.value())
+                .value(image.getValue())
                .type(imageType)
                .reason(image.buildReasonWithManualChangeDescriptions())
                .legalBasis(image.legalBasis())
@@ -341,6 +343,7 @@ public class EntityLogCreatorService {
            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
            case RECOMMENDATION -> EntryType.RECOMMENDATION;
            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+            case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
        };
    }

@@ -34,7 +34,8 @@ public class ManualChangeFactory {
            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
                    .withChange("type", recategorization.getType())
                    .withChange("section", recategorization.getSection())
-                    .withChange("legalBasis", recategorization.getLegalBasis());
+                    .withChange("legalBasis", recategorization.getLegalBasis())
+                    .withChange("value", recategorization.getValue());
        } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
@@ -594,6 +594,9 @@ public class EntityCreationService {
            throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
        }
        TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
+        if (trimmedTextRange.length() == 0){
+            return Optional.empty();
+        }
        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
        if (node.getEntities().contains(entity)) {
            Optional<TextEntity> optionalTextEntity = node.getEntities()
@@ -679,6 +682,8 @@ public class EntityCreationService {

        addEntityToGraph(mergedEntity, node);
        insertToKieSession(mergedEntity);
+
+        entitiesToMerge.stream().filter(e -> !e.equals(mergedEntity)).forEach(node.getEntities()::remove);
        return mergedEntity;
    }

@@ -763,26 +768,38 @@ public class EntityCreationService {
        try {
            if (node.getEntities().contains(entity)) {
                // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
-                node.getEntities()
+                Optional<TextEntity> optionalTextEntity = node.getEntities()
                        .stream()//
                        .filter(e -> e.equals(entity))//
                        .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
-                        .findAny()//
-                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
+                        .findAny();
+                if (optionalTextEntity.isPresent()) {
+                    addDuplicateEntityToGraph(optionalTextEntity.get(), entity.getTextRange(), node);
+                } else {
+                    node.getEntities().remove(entity);
+                    addNewEntityToGraph(entity, documentTree);
+                }
+
            } else {
                entity.addIntersectingNode(documentTree.getRoot().getNode());
                addEntityToGraph(entity, documentTree);
            }
        } catch (NoSuchElementException e) {
-            entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
-            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
-            entity.addIntersectingNode(documentTree.getRoot().getNode());
-            addToPages(entity);
-            addEntityToNodeEntitySets(entity);
+            addNewEntityToGraph(entity, documentTree);
        }
    }


+    private void addNewEntityToGraph(TextEntity entity, DocumentTree documentTree) {
+
+        entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
+        entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
+        entity.addIntersectingNode(documentTree.getRoot().getNode());
+        addToPages(entity);
+        addEntityToNodeEntitySets(entity);
+    }
+
+
    private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {

        entityToDuplicate.addTextRange(newTextRange);
@@ -1582,6 +1582,17 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+

 //------------------------------------ File attributes rules ------------------------------------

@@ -233,18 +233,13 @@ public class DictionaryServiceTest {

        dictionaryService.updateDictionary("dtId", "dossierId");
        var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId");
-        assertThat(dict.getDictionaryModels().size()).isEqualTo(1);
+        assertThat(dict.getDictionaryModels().size()).isEqualTo(2);
        var dictModel = dict.getDictionaryModels()
                .get(0);
        assertThat(dictModel.getType()).isEqualTo(type);
-        assertThat(dictModel.getEntries().size()).isEqualTo(4);
+        assertThat(dictModel.getEntries().size()).isEqualTo(3);
        dictModel.getEntries()
-                .forEach(entry -> {
-                    switch (entry.getValue()) {
-                        case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId());
-                        case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId());
-                    }
-                });
+                .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()));
    }

 }
@@ -177,7 +177,9 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
                                                                                              document,
                                                                                              TEST_DOSSIER_TEMPLATE_ID,
                                                                                              manualRedactions,
-                                                                                              TEST_FILE_ID);
+                                                                                              TEST_FILE_ID,
+                                                                                              Collections.emptySet(),
+                                                                                              false);

        redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
        assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
@@ -540,7 +540,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -558,7 +558,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

@@ -646,6 +646,62 @@ rule "PII.4.1: Redact line after contact information keywords"
            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
    end

+rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+

 // Rule unit: PII.5
 rule "PII.5.0: Redact line after contact information keywords reduced"
@@ -1191,21 +1247,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -1335,6 +1376,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -979,7 +979,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -997,7 +997,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

@@ -1037,6 +1037,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

+
 // Rule unit: PII.4
 rule "PII.4.0: Redact line after contact information keywords"
    when
@@ -1092,6 +1093,62 @@ rule "PII.4.1: Redact line after contact information keywords"
            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
    end

+rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+

 // Rule unit: PII.5
 rule "PII.5.0: Redact line after contact information keywords reduced"
@@ -1340,8 +1397,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

-
-// Rule unit: PII.12
 rule "PII.12.1: Expand PII entities with salutation prefix"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -1351,6 +1406,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix"
        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

+
 // Rule unit: PII.13
 rule "PII.13.0: Add recommendation for PII after Contact Person"
    when
@@ -1542,6 +1598,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
        $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
    end

+
 // Rule unit: ETC.9
 rule "ETC.9.0: Redact skipped impurities"
    when
@@ -1559,6 +1616,7 @@ rule "ETC.9.1: Redact impurities"
        $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
    end

+
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
    when
@@ -1567,6 +1625,7 @@ rule "ETC.10.0: Redact Product Composition Information"
        $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
    end

+
 // Rule unit: ETC.11
 rule "ETC.11.0: Recommend first line in table cell with name and address of owner"
    when
@@ -1596,6 +1655,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)"
        $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
    end

+
 //------------------------------------ AI rules ------------------------------------

 // Rule unit: AI.0
@@ -1641,7 +1701,8 @@ rule "AI.3.0: Recommend authors from AI as PII"
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
    end

-//------------------------------------ Manual redaction rules ------------------------------------
+
+//------------------------------------ Manual changes rules ------------------------------------

 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1761,7 +1822,6 @@ rule "MAN.3.2: Apply image recategorization"
        retract($recategorization);
    end

-
 rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
@@ -1770,6 +1830,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end

+
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
    salience 128
@@ -1808,21 +1869,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -1930,6 +1976,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
        retract($other);
    end

+
 // Rule unit: X.9
 rule "X.9.0: Merge mostly contained signatures"
    when
@@ -1940,6 +1987,7 @@ rule "X.9.0: Merge mostly contained signatures"
        $signature.addEngine(LayoutEngine.AI);
    end

+
 // Rule unit: X.10
 rule "X.10.0: remove false positives of ai"
    when
@@ -1949,6 +1997,33 @@ rule "X.10.0: remove false positives of ai"
        $aiSignature.remove("X.10.0", "Removed because false positive");
    end

+
+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -1437,6 +1437,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -296,6 +296,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
         });
    end

+
 // Rule unit: CBI.23
 rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
    when
@@ -315,6 +316,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

+
 //------------------------------------ PII rules ------------------------------------

 // Rule unit: PII.0
@@ -378,7 +380,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -396,7 +398,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

@@ -504,6 +506,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

+
 // Rule unit: PII.10
 rule "PII.10.0: Redact study director abbreviation"
    when
@@ -803,21 +806,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -947,6 +935,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -278,6 +278,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------

 // Rule unit: LDS.0
@@ -274,7 +274,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

-
 rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
@@ -602,7 +601,6 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve

 //------------------------------------ PII rules ------------------------------------

-
 // Rule unit: PII.0
 rule "PII.0.0: Redact all PII"
    when
@@ -734,6 +732,62 @@ rule "PII.4.1: Redact line after contact information keywords"
            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
    end

+rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+

 // Rule unit: PII.6
 rule "PII.6.0: Redact line between contact keywords"
@@ -1281,21 +1335,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -1425,6 +1464,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -258,21 +258,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -402,6 +387,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -523,6 +523,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -284,21 +284,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -439,6 +424,32 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -33,11 +33,7 @@ public class RuleFileMigrator {
            rulesToAdd.forEach(ruleFileBluePrint::addRule);
        }

-        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
-                                                               combinedBluePrint.globals(),
-                                                               combinedBluePrint.queries(),
-                                                               ruleFileBluePrint.ruleClasses());
-        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
+        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
        String migratedFilePath = ruleFile.getAbsolutePath();
        try (var out = new FileOutputStream(migratedFilePath)) {
            out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));
@@ -17,6 +17,7 @@ public class RuleType {
                                                    "ETC", "Other",//
                                                    "MAN", "Manual changes",//
                                                    "X", "Entity merging",//
+                                                    "DICT", "Dictionary merging",//
                                                    "FA", "File attributes",//
                                                    "LDS", "Local dictionary search",//
                                                    "TAB", "Table extraction",//
@@ -979,7 +979,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
    end

@@ -997,7 +997,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                              containsString("Fel") ||
                              containsString("Fer"))
    then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
    end

@@ -1092,6 +1092,62 @@ rule "PII.4.1: Redact line after contact information keywords"
            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
    end

+rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+

 // Rule unit: PII.5
 rule "PII.5.0: Redact line after contact information keywords reduced"
@@ -1808,21 +1864,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
    end


-// Rule unit: X.1
-rule "X.1.0: Merge intersecting Entities of same type"
-    salience 64
-    when
-        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
-    then
-        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
-        $first.remove("X.1.0", "merge intersecting Entities of same type");
-        $second.remove("X.1.0", "merge intersecting Entities of same type");
-        retract($first);
-        retract($second);
-        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
-    end
-

 // Rule unit: X.2
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@@ -1949,6 +1990,33 @@ rule "X.10.0: remove false positives of ai"
        $aiSignature.remove("X.10.0", "Removed because false positive");
    end

+
+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------

 // Rule unit: FA.1
@@ -1582,6 +1582,31 @@ rule "X.10.0: remove false positives of ai"
    end


+// Rule unit: X.11
+rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+    salience 64
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
+        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+    then
+        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        retract($dictionaryEntity);
+    end
+
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+

 //------------------------------------ File attributes rules ------------------------------------

@@ -8,5 +8,6 @@ X.5.0
 X.5.1
 X.6.*
 X.8.*
+DICT.*.*
 FA.*.*
 LDS.*.*
@@ -7,5 +7,6 @@ X.5.0
 X.5.1
 X.7.0
 X.8.*
+DICT.*.*
 FA.*.*
 LDS.*.*
@@ -9,5 +9,6 @@ ETC
 AI
 MAN
 X
+DICT
 FA
 LDS
Author	SHA1	Message	Date
Kilian Schüttler	b80914288b	Merge branch 'RED-7384-fp' into 'master' RED-7384: handle pending dict application in redaction-service instead of persistence Closes RED-7384 See merge request redactmanager/redaction-service!357	2024-04-03 16:32:08 +02:00
Kilian Schüttler	df694a90a8	RED-7384: handle pending dict application in redaction-service instead of persistence	2024-04-03 16:32:08 +02:00
Andrei Isvoran	49b91955b8	Merge branch 'RED-8877' into 'master' RED-8877 - Remove X.1.0 rule Closes RED-8877 See merge request redactmanager/redaction-service!361	2024-04-03 15:33:10 +02:00
Andrei Isvoran	04ec3cb7b9	RED-8877 - Remove X.1.0 rule	2024-04-03 15:33:10 +02:00
Kilian Schüttler	fcae77dcd4	Merge branch 'RED-8773' into 'master' RED-8773 - Wrong value for recategorized and forced logo Closes RED-8773 See merge request redactmanager/redaction-service!329	2024-04-03 15:18:06 +02:00
Corina Olariu	13e60e05b6	RED-8773 - Wrong value for recategorized and forced logo	2024-04-03 15:18:06 +02:00
Andrei Isvoran	90ba3239fc	Merge branch 'RED-8775' into 'master' RED-8775 - Add X.11 remove rule Closes RED-8775 See merge request redactmanager/redaction-service!340	2024-04-02 08:33:54 +02:00
Andrei Isvoran	6de0346052	RED-8775 - Add X.11 remove rule	2024-04-02 08:33:54 +02:00
Andrei Isvoran	7b520ee0ab	Merge branch 'RED-8828-fix-npe' into 'master' RED-8828 - Fix error when resizing dict based redaction Closes RED-8828 See merge request redactmanager/redaction-service!353	2024-04-02 08:33:43 +02:00
Andrei Isvoran	bf638dac01	RED-8828 - Fix error when resizing dict based redaction	2024-03-29 14:30:55 +02:00
Kilian Schüttler	9354cbb6bc	Merge branch 'hotfix_removechain' into 'master' Hotfix removechain See merge request redactmanager/redaction-service!349	2024-03-27 16:27:09 +01:00
Kilian Schüttler	406f5a48cf	Hotfix removechain	2024-03-27 16:27:09 +01:00
Christoph Schabert	a71ee61e1b	Update .gitlab-ci.yml	2024-03-27 15:41:37 +01:00
Andrei Isvoran	096544d8e2	Merge branch 'RED-8840-rules' into 'master' RED-8840 - Adjust rules Closes RED-8840 See merge request redactmanager/redaction-service!351	2024-03-27 15:33:49 +01:00
Andrei Isvoran	646f2020a8	RED-8840 - Adjust rules	2024-03-27 16:14:47 +02:00
Ali Oezyetimoglu	72a7761dd8	Merge branch 'RED-8480-B' into 'master' RED-8480: addded property "value" to places with recategorizations Closes RED-8480 See merge request redactmanager/redaction-service!348	2024-03-27 11:24:32 +01:00
Ali Oezyetimoglu	c869a624e6	RED-8480: addded property "value" to places with recategorizations	2024-03-27 11:10:59 +01:00
Dominique Eifländer	3ca37133c6	Merge branch 'RED-8834' into 'master' RED-8834: Fixed text entities with empty text range Closes RED-8834 See merge request redactmanager/redaction-service!345	2024-03-27 09:59:33 +01:00
Dominique Eifländer	93baf97ce3	RED-8834: Fixed text entities with empty text range	2024-03-27 09:47:56 +01:00
Andrei Isvoran	a0773d16dc	Merge branch 'hotfix-get-dictionary-npe' into 'master' Hotfix NPE on getDeepCopyDictionary See merge request redactmanager/redaction-service!341	2024-03-26 10:36:02 +01:00
Andrei Isvoran	c4816579c9	Hotfix NPE on getDeepCopyDictionary	2024-03-26 11:23:14 +02:00
Ali Oezyetimoglu	c9e80e3fa1	Merge branch 'RED-8840' into 'master' RED-8840 - Add PII.4 rules for sanitisation with correct legal basis Closes RED-8840 See merge request redactmanager/redaction-service!337	2024-03-25 16:33:07 +01:00
Andrei Isvoran	677d62deca	RED-8840 - Add PII.4 rules for sanitisation with correct legal basis	2024-03-25 16:33:07 +01:00
Kilian Schüttler	5229cd87f6	Merge branch 'RED-8610' into 'master' RED-8610: Dictionary remove on dossier level should be displayed as skipped Closes RED-8610 See merge request redactmanager/redaction-service!338	2024-03-25 14:13:44 +01:00
Kilian Schüttler	50e137575d	RED-8610: Dictionary remove on dossier level should be displayed as skipped	2024-03-25 14:13:44 +01:00
Kilian Schüttler	7303a7579b	Merge branch 'RED-8610' into 'master' RED-8610: Dictionary remove on dossier level should be displayed as skipped Closes RED-8610 See merge request redactmanager/redaction-service!335	2024-03-22 10:19:59 +01:00
Kilian Schüttler	2b4f174bdf	RED-8610: Dictionary remove on dossier level should be displayed as skipped	2024-03-22 10:19:59 +01:00