added imported flag

Pull request #341 : RED-3350 Communication with entity recognition via queues
Merge in RED/redaction-service from feature/RED-3350 to master * commit '68727178e79224b7ca7f9f293e48ca21b4ce2f91': RED-3350 Communication with entity recognition via queues
2022-02-11 11:33:34 +02:00 · 2022-02-09 15:24:47 +01:00 · 2022-02-09 15:19:55 +01:00 · 2022-02-01 11:19:26 +01:00 · 2022-02-01 11:06:32 +01:00 · 2022-01-31 14:47:54 +01:00
31 changed files with 964 additions and 758 deletions
@@ -1,4 +1,4 @@
-FROM red/base-image:1.0.0
+FROM red/redaction-service-base-v1:1.0.0

 ARG PLATFORM_JAR

@@ -7,12 +7,3 @@ ENV PLATFORM_JAR ${PLATFORM_JAR}
 ENV USES_ELASTICSEARCH false

 COPY ["${PLATFORM_JAR}", "/"]
-
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    wget cabextract xfonts-utils fonts-liberation \
- && rm -rf /var/lib/apt/lists/*
-
-RUN curl http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \
- && dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \
- && rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>0.151.0</persistence-service.version>
+        <persistence-service.version>1.39.0</persistence-service.version>
    </properties>

    <dependencies>
@@ -1,17 +1,17 @@
 package com.iqser.red.service.redaction.v1.model;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;

@Data
@Builder
@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImportedRedaction {
+
+    private String id;
+
+    @Builder.Default
+    private List<Rectangle> positions = new ArrayList<>();
+}
@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImportedRedactions {
+
+    @Builder.Default
+    private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
+}
@@ -0,0 +1,48 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.OffsetDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class ManualChange {
+
+    private AnnotationStatus annotationStatus;
+    private ManualRedactionType manualRedactionType;
+    private OffsetDateTime processedDate;
+    private String userId;
+    private Map<String, Object> propertyChanges = new HashMap<>();
+
+    public boolean isProcessed() {
+        return processedDate != null;
+    }
+
+    public static ManualChange from(BaseAnnotation baseAnnotation) {
+        ManualChange manualChange = new ManualChange();
+        manualChange.annotationStatus = baseAnnotation.getStatus();
+        manualChange.processedDate = baseAnnotation.getProcessedDate();
+        manualChange.userId = baseAnnotation.getUser();
+        return manualChange;
+    }
+
+    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
+        this.manualRedactionType = manualRedactionType;
+        return this;
+    }
+
+    public ManualChange withChange(String property, Object value) {
+        this.propertyChanges.put(property, value);
+        return this;
+    }
+
+}
@@ -1,5 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

 public enum ManualRedactionType {
-    ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE
+    ADD_LOCALLY,
+    ADD_TO_DICTIONARY,
+    REMOVE_LOCALLY,
+    REMOVE_FROM_DICTIONARY,
+    FORCE_REDACT,
+    FORCE_HINT,
+    RECATEGORIZE,
+    LEGAL_BASIS_CHANGE,
+    RESIZE
 }
@@ -2,6 +2,6 @@ package com.iqser.red.service.redaction.v1.model;

 public enum MessageType {

-    FULL_ANALYSE, REANALYSE, SURROUNDING_TEXT
+    ANALYSE, REANALYSE, STRUCTURE_ANALYSE, SURROUNDING_TEXT

 }
@@ -2,16 +2,9 @@ package com.iqser.red.service.redaction.v1.model;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
+import lombok.*;

-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;


@Data
@@ -28,27 +21,20 @@ public class RedactionLogEntry {
    private int matchedRule;
    private boolean rectangle;
    private String legalBasis;
+
+    private boolean imported;
+
    private boolean redacted;
    private boolean isHint;
    private boolean isRecommendation;

-    private boolean hasBeenResized;
-    private boolean hasBeenRecategorized;
-    private boolean hasLegalBasisChanged;
-    private boolean hasBeenForced;
-    private boolean hasBeenRemovedByManualOverride;
-
    private String section;
    private float[] color;

    @Builder.Default
    private List<Rectangle> positions = new ArrayList<>();
    private int sectionNumber;
-    private boolean manual;
-    private AnnotationStatus status;
-    private ManualRedactionType manualRedactionType;
-    private String manualRedactionUserId;
-    private boolean isDictionaryEntry;
+

    private String textBefore;
    private String textAfter;
@@ -62,21 +48,39 @@ public class RedactionLogEntry {
    private boolean isImage;
    private boolean imageHasTransparency;

+    private boolean isDictionaryEntry;
    private boolean isDossierDictionaryEntry;

    private boolean excluded;

-    private String recategorizationType;
-    private String legalBasisChangeValue;
-
    @EqualsAndHashCode.Exclude
    @Builder.Default
    private List<Change> changes = new ArrayList<>();

-    private Set<Engine> engines= new HashSet<>();
+    @EqualsAndHashCode.Exclude
+    @Builder.Default
+    private List<ManualChange> manualChanges = new ArrayList<>();
+
+    private Set<Engine> engines = new HashSet<>();

    private Set<String> reference = new HashSet<>();

+    public boolean lastChangeIsRemoved() {
+        return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
+    }

+    public boolean isLocalManualRedaction() {
+        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY &&
+                mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
+    }
+
+    public boolean isManuallyRemoved() {
+        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY &&
+                mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
+    }
+
+    private <T> Optional<T> last(List<T> list) {
+        return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
+    }

 }
@@ -55,8 +55,7 @@ public class RedactionController implements RedactionResource {

    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {

-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest
-                .getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
        var mergedRedactionLog = getRedactionLog(RedactionRequest.builder()
                .fileId(annotateRequest.getFileId())
                .manualRedactions(annotateRequest.getManualRedactions())
@@ -85,13 +84,11 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {

-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
        try {
            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);

-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                pdDocument.setAllSecurityToBeRemoved(true);

@@ -113,13 +110,11 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {

-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
        try {
            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);

-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                pdDocument.setAllSecurityToBeRemoved(true);

@@ -143,8 +138,7 @@ public class RedactionController implements RedactionResource {
        Document classifiedDoc;

        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
        } catch (Exception e) {
            throw new RedactionException(e);
@@ -186,8 +180,7 @@ public class RedactionController implements RedactionResource {

        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());

-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest
-                .getFileId());
+        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
        if (sectionGrid.getSections().isEmpty()) {

            log.info("SectionGrid does not have headlines set. Computing headlines now!");
@@ -196,8 +189,7 @@ public class RedactionController implements RedactionResource {
            // enhance section grid with headline data
            for (var sectionText : text.getSectionTexts()) {
                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText
-                                .getSectionAreas()
+                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
                                .stream()
                                .map(SectionArea::getPage)
                                .collect(Collectors.toSet()), sectionText.getSectionAreas()));
@@ -206,13 +198,7 @@ public class RedactionController implements RedactionResource {
        }

        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        if (redactionLog.getAnalysisVersion() == 0) {
-            // old redaction logs are returned directly
-            return redactionLog;
-        } else {
-            return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest
-                    .getManualRedactions(), redactionRequest.getExcludedPages());
-        }
+        return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
    }


@@ -234,11 +220,9 @@ public class RedactionController implements RedactionResource {
                                               @PathVariable("fileId") String fileId,
                                               @RequestBody ManualRedactions manualRedactions) {

-        long start = System.currentTimeMillis();
        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
-        log.info("add surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, System
-                .currentTimeMillis() - start);
-        return result;
+        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
+        return result.getManualRedactions();
    }

 }
@@ -1,5 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.queue;

+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -8,18 +15,10 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.NerAnalyserService;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
-
@Slf4j
@Service
@RequiredArgsConstructor
@@ -28,7 +27,6 @@ public class RedactionMessageReceiver {
    private final ObjectMapper objectMapper;
    private final AnalyzeService analyzeService;
    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
-    private final NerAnalyserService nerAnalyserService;
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;


@@ -40,31 +38,26 @@ public class RedactionMessageReceiver {
        log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
        AnalyzeResult result = null;

-        switch (analyzeRequest.getMessageType()){
+        switch (analyzeRequest.getMessageType()) {

            case REANALYSE:
                result = analyzeService.reanalyze(analyzeRequest);
                log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
                break;
-            case FULL_ANALYSE:
-                // TODO Seperate stucture analysis by other queue
-                analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-
-                // TODO NerEntities should be computed and stored in entity-recognition-service, should be triggered by a seperate queue after structure analysis
-                nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-
-                result = analyzeService.analyze(analyzeRequest);
-                log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result
-                        .getDuration());

+            case STRUCTURE_ANALYSE:
+                result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+                log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
                break;
+
+            case ANALYSE:
+                result = analyzeService.analyze(analyzeRequest);
+                log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                break;
+
            case SURROUNDING_TEXT:
-                var manualRedactions = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
-                result = AnalyzeResult.builder()
-                        .dossierId(analyzeRequest.getDossierId())
-                        .fileId(analyzeRequest.getFileId())
-                        .manualRedactions(manualRedactions)
-                        .build();
+                result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
+                log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
                break;
        }

@@ -12,6 +12,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -71,36 +72,59 @@ public class Section {
    private List<FileAttribute> fileAttributes = new ArrayList<>();


+    public void addAiEntities(String type, String asType) {

-    public void addAiEntities(String type, String asType){
+        Set<Entity> entitiesOfType = nerEntities.stream()
+                .filter(nerEntity -> nerEntity.getType().equals(type))
+                .collect(Collectors.toSet());
+        Set<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet());
+        Set<Entity> found = EntitySearchUtils.find(searchText, values, asType, headline, sectionNumber, false, false, Engine.NER, true);
+        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);

-        Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
-        entitiesOfType.forEach(nerEntity -> nerEntity.setType(asType));
-        EntitySearchUtils.clearAndFindPositions(entitiesOfType, searchableText, dictionary);
-        EntitySearchUtils.addEntitiesWithHigherRank(entities, entitiesOfType, dictionary);
+        Set<Entity> finalResult = new HashSet<>();
+
+        // Only keep Entities with correct offsets from AI Service.
+        Iterator<Entity> itty = found.iterator();
+        while (itty.hasNext()) {
+            Entity current = itty.next();
+            boolean foundSameOffsets = false;
+            for (Entity entity : entitiesOfType) {
+                if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
+                    foundSameOffsets = true;
+                }
+            }
+            if (foundSameOffsets) {
+                finalResult.add(current);
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, finalResult, dictionary);
        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
        nerEntities.removeAll(entitiesOfType);
    }


-    public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, int minPartMatches, boolean allowDuplicateTypes){
+    public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType,
+                               int minPartMatches, boolean allowDuplicateTypes) {

        Set<String> combineSet = Set.of(combineTypes.split(","));

-        List<Entity> sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
+        List<Entity> sorted = nerEntities.stream()
+                .sorted(Comparator.comparing(Entity::getStart))
+                .collect(Collectors.toList());
        Set<Entity> found = new HashSet<>();
        int start = -1;
        int lastEnd = -1;
        int numberOfMatchParts = 0;
        Set<String> foundParts = new HashSet<>();
-        for (Entity entity : sorted){
-            if(entity.getType().equals(startType) && start == -1) {
+        for (Entity entity : sorted) {
+            if (entity.getType().equals(startType) && start == -1) {
                lastEnd = entity.getEnd();
                start = entity.getStart();
                foundParts.add(entity.getType());
                numberOfMatchParts++;
-            } else if(!allowDuplicateTypes && foundParts.contains(entity.getType())){
-                if(numberOfMatchParts >= minPartMatches) {
+            } else if (!allowDuplicateTypes && foundParts.contains(entity.getType())) {
+                if (numberOfMatchParts >= minPartMatches) {
                    String value = searchText.substring(start, lastEnd);
                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
                }
@@ -108,14 +132,14 @@ public class Section {
                lastEnd = -1;
                numberOfMatchParts = 0;
                foundParts = new HashSet<>();
-                if(entity.getType().equals(startType)){
+                if (entity.getType().equals(startType)) {
                    lastEnd = entity.getEnd();
                    start = entity.getStart();
                    foundParts.add(entity.getType());
                    numberOfMatchParts++;
                }
-            } else if(entity.getType().equals(startType) && start != -1){
-                if(numberOfMatchParts >= minPartMatches) {
+            } else if (entity.getType().equals(startType) && start != -1) {
+                if (numberOfMatchParts >= minPartMatches) {
                    String value = searchText.substring(start, lastEnd);
                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
                }
@@ -125,24 +149,25 @@ public class Section {
                foundParts = new HashSet<>();
                foundParts.add(entity.getType());
                numberOfMatchParts++;
-            } else if(start != -1 && combineSet.contains(entity.getType()) && entity.getStart() - lastEnd < maxDistanceBetween){
+            } else if (start != -1 && combineSet.contains(entity.getType()) && entity.getStart() - lastEnd < maxDistanceBetween) {
                lastEnd = entity.getEnd();
                numberOfMatchParts++;
                foundParts.add(entity.getType());
            }
        }

-        if(numberOfMatchParts >= minPartMatches) {
+        if (numberOfMatchParts >= minPartMatches) {
            String value = searchText.substring(start, lastEnd);
            found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
        }

-        if(!found.isEmpty()) {
+        if (!found.isEmpty()) {
            EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
        }
    }

+
    @WhenCondition
    public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
                                           @Argument(ArgumentType.STRING) String value) {
@@ -235,6 +260,7 @@ public class Section {
        return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
    }

+
    @WhenCondition
    public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {

@@ -257,7 +283,8 @@ public class Section {


    @ThenAction
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.REGEX) String suffixPattern,
                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                              @Argument(ArgumentType.INTEGER) int group) {

@@ -266,7 +293,8 @@ public class Section {


    @ThenAction
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.REGEX) String suffixPattern,
                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                              @Argument(ArgumentType.INTEGER) int group,
                              @Argument(ArgumentType.REGEX) String valuePattern) {
@@ -378,7 +406,8 @@ public class Section {
    }


-    public void ignore(String type){
+    public void ignore(String type) {
+
        entities.removeIf(entity -> entity.getType().equals(type));
    }

@@ -702,19 +731,13 @@ public class Section {
    }


-    private Set<Entity> findEntities(String value,
-                                     String asType,
-                                     boolean caseInsensitive,
-                                     boolean redacted,
-                                     int ruleNumber,
-                                     String reason,
-                                     String legalBasis,
-                                     Engine engine) {
+    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
+                                     int ruleNumber, String reason, String legalBasis, Engine engine) {

        String text = caseInsensitive ? searchText.toLowerCase() : searchText;
        String searchValue = caseInsensitive ? value.toLowerCase() : value;

-        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, engine);
+        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, engine, false);

        found.forEach(entity -> {
            if (redacted) {
@@ -1,31 +1,55 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
+import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;

@Slf4j
@Service
@@ -43,11 +67,11 @@ public class AnalyzeService {
    private final RedactionServiceSettings redactionServiceSettings;
    private final SectionTextBuilderService sectionTextBuilderService;
    private final SectionGridCreatorService sectionGridCreatorService;
-    private final NerAnalyserService nerAnalyserService;
    private final ImageService imageService;
+    private final ImportedRedactionService importedRedactionService;


-    public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
+    public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();

@@ -55,9 +79,7 @@ public class AnalyzeService {
        Document classifiedDoc;

        try {
-
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
-                    .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));

            Map<Integer, List<PdfImage>> pdfImages = null;
            if (redactionServiceSettings.isEnableImageClassification()) {
@@ -75,50 +97,24 @@ public class AnalyzeService {
        Text text = new Text(pageCount, sectionTexts);

        // enhance section grid with headline data
-        sectionTexts.forEach(sectionText -> classifiedDoc
-                .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                        sectionText.getHeadline(),
-                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                        sectionText.getSectionAreas())));
+        sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid()
+                .getSections()
+                .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
+                        .stream()
+                        .map(SectionArea::getPage)
+                        .collect(Collectors.toSet()), sectionText.getSectionAreas())));

+        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
-                .getSectionGrid());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());

-        log.info("Document structure analysis successful, took: {}", System.currentTimeMillis() - startTime);
-    }
-
-
-    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
-
-        long startTime = System.currentTimeMillis();
-
-        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
-            nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        }
-
-        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
-        long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
-                .getDossierId());
-
-        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
-
-        dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
-                .getDossierTemplateId());
-
-        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
-                redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
-                rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
+        return AnalyzeResult.builder()
+                .dossierId(analyzeRequest.getDossierId())
+                .fileId(analyzeRequest.getFileId())
+                .duration(System.currentTimeMillis() - startTime)
+                .numberOfPages(text.getNumberOfPages())
+                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
+                .build();
    }


@@ -135,8 +131,7 @@ public class AnalyzeService {
            return analyze(analyzeRequest);
        }

-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
-                .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());

        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
                .isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
@@ -145,10 +140,11 @@ public class AnalyzeService {
            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
        }

-        var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
-            nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = NerEntities.builder().build();
        }

        List<SectionText> reanalysisSections = text.getSectionTexts()
@@ -158,19 +154,56 @@ public class AnalyzeService {

        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
-                .getDossierId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());

        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
-        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
-                .getDossierTemplateId());
+        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
+
+        redactionLog.getRedactionLogEntry()
+                .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
+                        .equals(IMPORTED_REDACTION_TYPE));
+        redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);

-        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
-        redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
    }


+    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = NerEntities.builder().build();
+        }
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+        long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
+
+        dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+
+        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, legalBasis, dictionary.getVersion()
+                .getDossierTemplateVersion(), dictionary.getVersion()
+                .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog.getRedactionLogEntry(), true);
+        redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
+
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
+    }
+
+
    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog,
                                                 Text text, AnalyzeRequest analyzeRequest) {

@@ -180,7 +213,7 @@ public class AnalyzeService {
        Set<Integer> sectionsToReanalyse = new HashSet<>();
        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
                sectionsToReanalyse.add(entry.getSectionNumber());
            }
            if (entry.isImage()) {
@@ -211,10 +244,8 @@ public class AnalyzeService {

        excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());

-        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
-                redactionLog, analyzeRequest.getAnalysisNumber());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
-                .getRedactionLog());
+        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog, analyzeRequest.getAnalysisNumber());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());

        long duration = System.currentTimeMillis() - startTime;

@@ -231,6 +262,7 @@ public class AnalyzeService {
                .legalBasisVersion(redactionLog.getLegalBasisVersion())
                .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
                .wasReanalyzed(isReanalysis)
+                .manualRedactions(analyzeRequest.getManualRedactions())
                .build();
    }

@@ -242,13 +274,14 @@ public class AnalyzeService {
        }

        return Stream.concat(manualRedactions.getLegalBasisChanges()
-                        .stream()
-                        .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
-                        .stream()
-                        .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
-                        .stream()
-                        .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
-                .collect(Collectors.toSet());
+                .stream()
+                .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
+                .stream()
+                .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
+                .stream()
+                .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions()
+                .stream()
+                .map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet());
    }


@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
 import lombok.RequiredArgsConstructor;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -15,13 +16,10 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import java.awt.Color;
+import java.awt.*;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;

@Service
@@ -107,7 +105,7 @@ public class AnnotationService {

    private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {

-        if (redactionLogEntry.isManual()) {
+        if (redactionLogEntry.isLocalManualRedaction()) {
            return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
        }
        return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
@@ -6,7 +6,6 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -14,7 +13,8 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;

-import java.awt.Color;
+import java.awt.*;
+import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -63,7 +63,7 @@ public class DictionaryService {
            });
        });

-        if(dictionariesByDossier.containsKey(dossierId)) {
+        if (dictionariesByDossier.containsKey(dossierId)) {
            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
            dictionaryModels.forEach(dictionaryModel -> {
                dictionaryModel.getEntries().forEach(dictionaryEntry -> {
@@ -105,7 +105,7 @@ public class DictionaryService {
                dictionaryRepresentation.setDictionaryVersion(version);
                dictionaryRepresentation.setDictionary(dictionary);

-                if(dossierId == null) {
+                if (dossierId == null) {
                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
                } else {
                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
@@ -204,7 +204,7 @@ public class DictionaryService {

        //TODO merge dictionaries if they have same names
        long dossierDictionaryVersion = -1;
-        if(dictionariesByDossier.containsKey(dossierId)) {
+        if (dictionariesByDossier.containsKey(dossierId)) {
            var dossierRepresentation = dictionariesByDossier.get(dossierId);
            dossierRepresentation.getDictionary().forEach(dm -> {
                copy.add(SerializationUtils.clone(dm));
@@ -9,15 +9,13 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
-
-import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;

+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -62,7 +60,7 @@ public class EntityRedactionService {
            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
        }

-        Map<Integer, List<Entity>> entitiesPerPage = convertToEnititesPerPage(entities);
+        Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
        return new PageEntities(entitiesPerPage, imagesPerPage);
    }

@@ -161,7 +159,7 @@ public class EntityRedactionService {
    }


-    private Map<Integer, List<Entity>> convertToEnititesPerPage(Set<Entity> entities) {
+    private Map<Integer, List<Entity>> convertToEntitiesPerPage(Set<Entity> entities) {

        Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
        for (Entity entity : entities) {
@@ -225,7 +223,7 @@ public class EntityRedactionService {

    private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
                                  Dictionary dictionary, boolean local, NerEntities nerEntities,
-                                  List<Integer> cellstarts) {
+                                  List<Integer> cellStarts) {

        Set<Entity> found = new HashSet<>();
        String searchableString = searchableText.toString();
@@ -237,52 +235,46 @@ public class EntityRedactionService {
        for (DictionaryModel model : dictionary.getDictionaryModels()) {
            if (model.isCaseInsensitive()) {
                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model
-                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
+                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false));
            } else {
                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(searchableString, model.getValues(local), model
-                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
+                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false));
            }
        }

        Set<Entity> nerFound = new HashSet<>();
        if (!local) {
-            Map<String, Set<String>> nerValuesPerType = getNerValues(sectionNumber, nerEntities, cellstarts);
-            nerValuesPerType.entrySet().forEach(entry -> {
-                EntitySearchUtils.addEntitiesWithHigherRank(nerFound, EntitySearchUtils.find(searchableString, entry.getValue(), entry
-                        .getKey(), headline, sectionNumber, false, false, Engine.NER), dictionary);
-            });
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
        }

-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound) ;
+        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
    }


-    private Map<String, Set<String>> getNerValues(int sectionNumber, NerEntities nerEntities,
-                                                  List<Integer> cellstarts) {
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts,
+                                     String headline) {

-        Map<String, Set<String>> nerValuesPerType = new HashMap<>();
+        Set<Entity> entities = new HashSet<>();

-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities.getResult()
-                .containsKey(sectionNumber)) {
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getResult().containsKey(sectionNumber)) {
            nerEntities.getResult().get(sectionNumber).forEach(res -> {
-                if (cellstarts == null || cellstarts.isEmpty()) {
-                    nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
-                            .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
                } else {
                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellstarts) {
+                    for (Integer cellStart : cellStarts) {
                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
                            intersectsCellStart = true;
+                            break;
                        }
                    }
                    if (!intersectsCellStart) {
-                        nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
-                                .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
+                        entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
                    }
                }
            });
        }
-        return nerValuesPerType;
+        return entities;
    }

 }
@@ -0,0 +1,102 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
+import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ImportedRedactionService {
+
+    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+
+    private final DictionaryService dictionaryService;
+    private final RedactionStorageService redactionStorageService;
+
+
+    public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
+                                          List<RedactionLogEntry> redactionLogEntries, boolean addImportedRedactions) {
+
+        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
+        if (importedRedactions == null) {
+            return redactionLogEntries;
+        }
+
+        redactionLogEntries.removeIf(redactionLogEntry -> hasIntersections(redactionLogEntry, importedRedactions));
+
+        if (addImportedRedactions) {
+            return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
+                                                          List<RedactionLogEntry> redactionLogEntries,
+                                                          ImportedRedactions importedRedactions) {
+
+        for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (ImportedRedaction importedRedaction : importedRedactionsValues) {
+                RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
+                        .id(importedRedaction.getId())
+                        .type(IMPORTED_REDACTION_TYPE)
+                        .imported(true)
+                        .redacted(true)
+                        .positions(importedRedaction.getPositions())
+                        .color(getColor("imported_redaction", dossierTemplateId))
+                        .build();
+
+                redactionLogEntries.add(redactionLogEntry);
+            }
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    private boolean hasIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+
+        for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
+                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
+                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
+                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                        if (intersects(importedRedactionPosition, rectangle)) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
+
+    private boolean intersects(Rectangle importedPosition, Rectangle redactionLogPosition) {
+
+        return redactionLogPosition.getTopLeft()
+                .getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeft()
+                .getX() && redactionLogPosition.getTopLeft()
+                .getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeft()
+                .getY() && redactionLogPosition.getTopLeft().getX() < importedPosition.getTopLeft()
+                .getX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
+                .getY() < importedPosition.getTopLeft().getY() + importedPosition.getHeight();
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId) {
+
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+}
@@ -7,10 +7,11 @@ import java.util.Set;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -33,8 +34,9 @@ public class ManualRedactionSurroundingTextService {
    private final SurroundingWordsService surroundingWordsService;


-    public ManualRedactions addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
+    public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {

+        long startTime = System.currentTimeMillis();
        Text text = redactionStorageService.getText(dossierId, fileId);
        List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
        List<ManualResizeRedaction> processedResizeRedactions = new ArrayList<>();
@@ -51,8 +53,7 @@ public class ManualRedactionSurroundingTextService {
                while (addItty.hasNext()) {
                    var manualAddRedaction = addItty.next();
                    if (sectionContainsEntry(sectionArea, manualAddRedaction.getPositions())) {
-                        var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction
-                                .getPositions());
+                        var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction.getPositions());
                        manualAddRedaction.setTextBefore(surroundingText.getLeft());
                        manualAddRedaction.setTextAfter(surroundingText.getRight());
                        processedAddRedactions.add(manualAddRedaction);
@@ -64,8 +65,7 @@ public class ManualRedactionSurroundingTextService {
                while (resizeItty.hasNext()) {
                    var manualResizeRedaction = resizeItty.next();
                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
-                        var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction
-                                .getPositions());
+                        var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                        manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                        manualResizeRedaction.setTextAfter(surroundingText.getRight());
                        processedResizeRedactions.add(manualResizeRedaction);
@@ -77,27 +77,31 @@ public class ManualRedactionSurroundingTextService {

        manualRedactions.getEntriesToAdd().addAll(processedAddRedactions);
        manualRedactions.getResizeRedactions().addAll(processedResizeRedactions);
-        return manualRedactions;
+
+        return AnalyzeResult.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .manualRedactions(manualRedactions)
+                .duration(System.currentTimeMillis() - startTime)
+                .build();
    }


    private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
                                                     List<Rectangle> toFindPositions) {

-        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText
-                .getSectionNumber(), false, false, Engine.DICTIONARY);
+        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, false);
        Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);

        Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);

-        if(correctEntity == null){
+        if (correctEntity == null) {
            log.warn("Could not calculate surrounding text");
-            return Pair.of("","");
+            return Pair.of("", "");
        }

        if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText
-                    .getCellStarts());
+            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
        } else {
            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
        }
@@ -1,53 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.codec.binary.Base64;
-import org.springframework.stereotype.Service;
-
-import java.util.stream.Collectors;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class NerAnalyserService {
-
-    private final RedactionStorageService redactionStorageService;
-    private final EntityRecognitionClient entityRecognitionClient;
-    private final RedactionServiceSettings redactionServiceSettings;
-
-    public void computeNerEntities(String dossierId, String fileId) {
-
-        if (redactionServiceSettings.isEnableEntityRecognition()) {
-            var text = redactionStorageService.getText(dossierId, fileId);
-
-            long start = System.currentTimeMillis();
-
-            if (text != null) {
-                var nerRequest = EntityRecognitionRequest.builder()
-                        .data(text.getSectionTexts()
-                                .stream()
-                                .map(sectionText -> new EntityRecognitionSection(sectionText.getSectionNumber(), new String(Base64
-                                        .encodeBase64(sectionText
-                                                .getText().getBytes()))))
-                                .collect(Collectors.toList()))
-                        .build();
-
-                var nerResponse = entityRecognitionClient.findAuthors(nerRequest);
-
-                log.info("Computing NER entities took: {} ms for dossierId {} and fileId {}", System.currentTimeMillis() - start, dossierId, fileId);
-
-                redactionStorageService.storeObject(dossierId, fileId, FileType.NER_ENTITIES, nerResponse);
-            } else {
-                log.warn("Warning, text for file: {} in dossier: {} is null", fileId, dossierId);
-            }
-        }
-    }
-
-}
@@ -33,17 +33,11 @@ public class RedactionChangeLogService {

        List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
                .stream()
-                .filter(entry -> !entry.getChanges()
-                        .get(entry.getChanges().size() - 1)
-                        .getType()
-                        .equals(ChangeType.REMOVED))
+                .filter(entry -> !entry.lastChangeIsRemoved())
                .collect(Collectors.toList());

        Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
-                .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges()
-                        .get(entry.getChanges().size() - 1)
-                        .getType()
-                        .equals(ChangeType.REMOVED)).collect(Collectors.toSet());
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()).collect(Collectors.toSet());

        notRemovedPreviousEntries.forEach(added::remove);

@@ -87,7 +81,7 @@ public class RedactionChangeLogService {

        addedIds.forEach((k, v) -> {
            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now()));
+                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
            }
            newRedactionLogEntries.add(v);
        });
@@ -70,7 +70,6 @@ public class RedactionLogCreatorService {
                    .legalBasis(image.getLegalBasis())
                    .matchedRule(image.getMatchedRule())
                    .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
-                    .manual(false)
                    .isDictionaryEntry(false)
                    .isRecommendation(false)
                    .positions(List.of(new Rectangle(new Point((float) image.getPosition()
@@ -92,7 +91,7 @@ public class RedactionLogCreatorService {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

-        // Duplicates can exist due table extraction colums over multiple rows.
+        // Duplicates can exist due table extraction columns over multiple rows.
        Set<String> processedIds = new HashSet<>();

        entityLoop:
@@ -173,10 +172,7 @@ public class RedactionLogCreatorService {
    private boolean isCharClose(float x, float xDirAdj, float width) {

        float max = x + (5 * width);
-        if (xDirAdj < max) {
-            return true;
-        }
-        return false;
+        return xDirAdj < max;
    }


@@ -190,11 +186,7 @@ public class RedactionLogCreatorService {
        float minY = y - offsetY;
        float maxY = y + offsetY;

-        if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) {
-            return true;
-        }
-
-        return false;
+        return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight;
    }


@@ -1,7 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
 import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -46,7 +48,7 @@ public class RedactionLogMergeService {

                if (excludedPages != null && !excludedPages.isEmpty()) {
                    entry.getPositions().forEach(pos -> {
-                        if (!entry.isManual() && excludedPages.contains(pos.getPage())) {
+                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
                            entry.setExcluded(true);
                        }
                    });
@@ -117,9 +119,9 @@ public class RedactionLogMergeService {
                var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
                String manualOverrideReason = null;
                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
+
+
                    redactionLogEntry.setType(imageRecategorization.getType());
-                    redactionLogEntry.setHasBeenRecategorized(true);
                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());

                    if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
@@ -132,77 +134,65 @@ public class RedactionLogMergeService {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
-                    redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
+
+                redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
+                        .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                        .withChange("type", imageRecategorization.getType()));
            }

            if (mrw.getItem() instanceof IdRemoval) {
                var manualRemoval = (IdRemoval) mrw.getItem();
                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
-                    log.info("Skipping merge for dictionary-modifying entry");
+                    log.debug("Skipping merge for dictionary-modifying entry");
                } else {
                    String manualOverrideReason = null;
                    if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
                        redactionLogEntry.setRedacted(false);
-                        redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
                        redactionLogEntry.setHint(false);
-                        redactionLogEntry.setHasBeenRemovedByManualOverride(true);
                    } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
-                    } else {
-                        redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                    }

                    redactionLogEntry.setReason(manualOverrideReason);
-                    redactionLogEntry.setManual(true);
-                    redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser());
-                    redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
-                    redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
-                    redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
+
+                    redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
+                            .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
+                                    ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
                }
            }

            if (mrw.getItem() instanceof ManualForceRedaction) {
                var manualForceRedact = (ManualForceRedaction) mrw.getItem();
                String manualOverrideReason = null;
+                var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
                if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
                    // Forcing a skipped hint should result in a hint
-                    if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) {
+                    if (dictionaryIsHint) {
                        redactionLogEntry.setHint(true);
                    } else {
                        redactionLogEntry.setRedacted(true);
                    }
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-                    redactionLogEntry.setHasBeenForced(true);
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
+                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                var manualChange = ManualChange.from(manualForceRedact)
+                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+
+                redactionLogEntry.getManualChanges().add(manualChange);

            }

@@ -210,11 +200,9 @@ public class RedactionLogMergeService {
                var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
                String manualOverrideReason = null;
                if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
                    redactionLogEntry.setRedacted(true);
-                    redactionLogEntry.setHasLegalBasisChanged(true);
                    if (manualLegalBasisChange.getSection() != null) {
                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
                    }
@@ -223,47 +211,45 @@ public class RedactionLogMergeService {
                    }
                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
-                    redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
+
+                var manualChange = ManualChange.from(manualLegalBasisChange)
+                        .withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
+                manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
+                if (manualLegalBasisChange.getSection() != null) {
+                    manualChange.withChange("section", manualLegalBasisChange.getSection());
+                }
+                if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
+                    manualChange.withChange("value", manualLegalBasisChange.getValue());
+                }
+                redactionLogEntry.getManualChanges().add(manualChange);
            }

            if (mrw.getItem() instanceof ManualResizeRedaction) {
                var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
                String manualOverrideReason = null;
                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                    redactionLogEntry.setValue(manualResizeRedact.getValue());
-                    redactionLogEntry.setHasBeenResized(true);
                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE);
-
+                redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
+                        .withManualRedactionType(ManualRedactionType.RESIZE)
+                        .withChange("value", manualResizeRedact.getValue()));
            }

        });
@@ -319,14 +305,22 @@ public class RedactionLogMergeService {

    private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {

-        return manualRedactionEntry.getStatus()
-                .equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
+        return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
+                manualRedactionEntry.getProcessedDate() != null &&
+                (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
    }


    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
                                                      String dossierTemplateId) {

+        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
+
+        var change = ManualChange.from(manualRedactionEntry)
+                .withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
+        List<ManualChange> changeList = new ArrayList<>();
+        changeList.add(change);
+
        return RedactionLogEntry.builder()
                .id(id)
                .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
@@ -340,11 +334,8 @@ public class RedactionLogMergeService {
                .redacted(true)
                .isHint(false)
                .sectionNumber(-1)
-                .manual(true)
-                .status(manualRedactionEntry.getStatus())
-                .manualRedactionType(ManualRedactionType.ADD)
-                .manualRedactionUserId(manualRedactionEntry.getUser())
                .rectangle(manualRedactionEntry.isRectangle())
+                .manualChanges(changeList)
                .build();
    }

@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
@@ -14,8 +9,11 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;

@Service
@RequiredArgsConstructor
@@ -1,35 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
-import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.classification.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;

@Slf4j
@Service
@@ -55,7 +55,7 @@ public class EntitySearchUtils {


    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine) {
+                            boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine, boolean ignoreMinLength) {

        Set<Entity> found = new HashSet<>();

@@ -63,7 +63,7 @@ public class EntitySearchUtils {

            String cleanValue = value.trim();

-            if (cleanValue.length() <= 2) {
+            if (!ignoreMinLength && cleanValue.length() <= 2) {
                continue;
            }

@@ -128,7 +128,12 @@ public class EntitySearchUtils {
            for (Entity inner : entities) {
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                    wordsToRemove.add(inner);
+                    // FIXME this is workaround for RED-3327 and should be removed in the future.
+                    if(word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
+                        wordsToRemove.add(word);
+                    } else {
+                        wordsToRemove.add(inner);
+                    }
                }
            }
        }
@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.settings;

-import lombok.Data;
 import org.springframework.boot.context.properties.ConfigurationProperties;

+import lombok.Data;
+
@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
@@ -17,6 +18,6 @@ public class RedactionServiceSettings {

    private int analysisVersion = 1;

-    private boolean enableEntityRecognition = true;
+    private boolean nerServiceEnabled = true;

 }
@@ -8,6 +8,7 @@ import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
@@ -44,6 +45,24 @@ public class RedactionStorageService {
    }


+    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
+
+        InputStreamResource inputStreamResource;
+        try {
+            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Imported redactions not available.");
+            return null;
+        }
+
+        try {
+            return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not imported redactions", e);
+        }
+    }
+
+
    public RedactionLog getRedactionLog(String dossierId, String fileId) {

        InputStreamResource inputStreamResource;
@@ -86,14 +105,13 @@ public class RedactionStorageService {
        try {
            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
        } catch (StorageObjectDoesNotExist e) {
-            log.debug("NER Entities not available.");
-            return null;
+            throw new NotFoundException("NER Entities are not available.");
        }

        try {
            return objectMapper.readValue(inputStreamResource.getInputStream(), NerEntities.class);
        } catch (IOException e) {
-            throw new RuntimeException("Could not convert NerEntities", e);
+            throw new RuntimeException("Could not convert NER Entities", e);
        }
    }

@@ -127,8 +145,6 @@ public class RedactionStorageService {
            return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
        }

-
-
    }

 }
@@ -51,14 +51,14 @@ import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
@@ -113,6 +113,7 @@ public class RedactionIntegrationTest {
    private static final String FORMULA = "formula";
    private static final String OCR = "ocr";
    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";

    private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
    private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -217,6 +218,7 @@ public class RedactionIntegrationTest {

        loadDictionaryForTest();
        loadTypeForTest();
+        loadNerForTest();
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getTypeResponse());

@@ -253,291 +255,11 @@ public class RedactionIntegrationTest {
        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
    }


-    private void loadDictionaryForTest() {
-
-        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(RECOMMENDATION_AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/recommendation_CBI_author.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(RECOMMENDATION_ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/recommendation_CBI_address.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FALSE_POSITIVE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/false_positive.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
-    }
-
-
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
-    private void loadTypeForTest() {
-
-        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#fab4c0");
-        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
-        typeColorMap.put(TEST_METHOD, "#91fae8");
-        typeColorMap.put(PII, "#66ccff");
-        typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
-        typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
-        typeColorMap.put(FALSE_POSITIVE, "#ffffff");
-        typeColorMap.put(PURITY, "#ffe187");
-        typeColorMap.put(IMAGE, "#fcc5fb");
-        typeColorMap.put(OCR, "#fcc5fb");
-        typeColorMap.put(LOGO, "#ffe187");
-        typeColorMap.put(FORMULA, "#ffe187");
-        typeColorMap.put(SIGNATURE, "#ffe187");
-
-        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(ADDRESS, false);
-        hintTypeMap.put(AUTHOR, false);
-        hintTypeMap.put(SPONSOR, false);
-        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(REDACTION_INDICATOR, true);
-        hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
-        hintTypeMap.put(PUBLISHED_INFORMATION, true);
-        hintTypeMap.put(TEST_METHOD, true);
-        hintTypeMap.put(PII, false);
-        hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
-        hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
-        hintTypeMap.put(FALSE_POSITIVE, true);
-        hintTypeMap.put(PURITY, false);
-        hintTypeMap.put(IMAGE, true);
-        hintTypeMap.put(OCR, true);
-        hintTypeMap.put(FORMULA, false);
-        hintTypeMap.put(LOGO, false);
-        hintTypeMap.put(SIGNATURE, false);
-        hintTypeMap.put(DOSSIER_REDACTIONS, false);
-
-        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(AUTHOR, false);
-        caseInSensitiveMap.put(SPONSOR, false);
-        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
-        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
-        caseInSensitiveMap.put(TEST_METHOD, false);
-        caseInSensitiveMap.put(PII, false);
-        caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
-        caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
-        caseInSensitiveMap.put(FALSE_POSITIVE, false);
-        caseInSensitiveMap.put(PURITY, false);
-        caseInSensitiveMap.put(IMAGE, true);
-        caseInSensitiveMap.put(OCR, true);
-        caseInSensitiveMap.put(SIGNATURE, true);
-        caseInSensitiveMap.put(LOGO, true);
-        caseInSensitiveMap.put(FORMULA, true);
-        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
-
-        recommendationTypeMap.put(VERTEBRATE, false);
-        recommendationTypeMap.put(ADDRESS, false);
-        recommendationTypeMap.put(AUTHOR, false);
-        recommendationTypeMap.put(SPONSOR, false);
-        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(HINT_ONLY, false);
-        recommendationTypeMap.put(MUST_REDACT, false);
-        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
-        recommendationTypeMap.put(TEST_METHOD, false);
-        recommendationTypeMap.put(PII, false);
-        recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
-        recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
-        recommendationTypeMap.put(FALSE_POSITIVE, false);
-        recommendationTypeMap.put(PURITY, false);
-        recommendationTypeMap.put(IMAGE, false);
-        recommendationTypeMap.put(OCR, false);
-        recommendationTypeMap.put(FORMULA, false);
-        recommendationTypeMap.put(SIGNATURE, false);
-        recommendationTypeMap.put(LOGO, false);
-        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
-
-        rankTypeMap.put(FALSE_POSITIVE, 160);
-        rankTypeMap.put(PURITY, 155);
-        rankTypeMap.put(PII, 150);
-        rankTypeMap.put(ADDRESS, 140);
-        rankTypeMap.put(AUTHOR, 130);
-        rankTypeMap.put(SPONSOR, 120);
-        rankTypeMap.put(VERTEBRATE, 110);
-        rankTypeMap.put(MUST_REDACT, 100);
-        rankTypeMap.put(REDACTION_INDICATOR, 90);
-        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
-        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
-        rankTypeMap.put(TEST_METHOD, 60);
-        rankTypeMap.put(HINT_ONLY, 50);
-        rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
-        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
-        rankTypeMap.put(IMAGE, 30);
-        rankTypeMap.put(OCR, 29);
-        rankTypeMap.put(LOGO, 28);
-        rankTypeMap.put(SIGNATURE, 27);
-        rankTypeMap.put(FORMULA, 26);
-        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
-
-        colors.setDefaultColor("#acfc00");
-        colors.setNotRedacted("#cccccc");
-        colors.setRequestAdd("#04b093");
-        colors.setRequestRemove("#04b093");
-    }
-
-
-    private List<Type> getTypeResponse() {
-
-        return typeColorMap.entrySet()
-                .stream()
-                .map(typeColor -> Type.builder()
-                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                        .type(typeColor.getKey())
-                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                        .hexColor(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey()))
-                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
-                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
-                        .rank(rankTypeMap.get(typeColor.getKey()))
-                        .build())
-
-                .collect(Collectors.toList());
-    }
-
-
-    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
-
-        return Type.builder()
-                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .hexColor(typeColorMap.get(type))
-                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
-                .isHint(hintTypeMap.get(type))
-                .isCaseInsensitive(caseInSensitiveMap.get(type))
-                .isRecommendation(recommendationTypeMap.get(type))
-                .rank(rankTypeMap.get(type))
-                .build();
-    }
-
-
-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-
-        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
-        entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder()
-                    .value(entry)
-                    .version(reanlysisVersions.getOrDefault(entry, 0L))
-                    .deleted(deleted.contains(entry))
-                    .build());
-        });
-        return dictionaryEntries;
-    }
-
-
    @Test
    public void test270Rotated() {

@@ -579,9 +301,7 @@ public class RedactionIntegrationTest {
            duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
        });

-        duplicates.entrySet().forEach(entry -> {
-            assertThat(entry.getValue().size()).isEqualTo(1);
-        });
+        duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));

        dictionary.get(AUTHOR).add("Drinking water");
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
@@ -846,6 +566,7 @@ public class RedactionIntegrationTest {
        System.out.println("Output file:" + outputFileName);
    }

+
    @Test
    public void testChangeComputation() throws IOException {

@@ -861,24 +582,27 @@ public class RedactionIntegrationTest {
        dictionary.get(AUTHOR).add("report");
        reanlysisVersions.put("report", 2L);
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
        analyzeService.reanalyze(request);

        dictionary.get(AUTHOR).add("assessment report");
        reanlysisVersions.put("assessment report", 3L);
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
        analyzeService.reanalyze(request);

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
        analyzeService.reanalyze(request);

-        var  redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);

-        var changes = redactionLog.getRedactionLogEntry().stream().filter(entry ->
-                entry.getValue() != null && entry.getValue().equals("report"))
-                .findFirst().get().getChanges();
+        var changes = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
+                .findFirst()
+                .get()
+                .getChanges();

        assertThat(changes.size()).isEqualTo(2);

@@ -892,6 +616,7 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void redactionTest() throws IOException {

@@ -1037,8 +762,6 @@ public class RedactionIntegrationTest {
    @Test
    public void testManualRedaction() throws IOException {

-//        675eba69b0c2917de55462c817adaa05
-
        System.out.println("testManualRedaction");
        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@@ -1089,8 +812,6 @@ public class RedactionIntegrationTest {
                .page(1)
                .build()));

-//        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
-
        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
        request.setManualRedactions(manualRedactions);
        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
@@ -1236,34 +957,6 @@ public class RedactionIntegrationTest {
    }


-    @SneakyThrows
-    private AnalyzeRequest prepareStorage(String file) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-
-        return prepareStorage(pdfFileResource.getInputStream());
-    }
-
-
-    @SneakyThrows
-    private AnalyzeRequest prepareStorage(InputStream stream) {
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .lastProcessed(OffsetDateTime.now())
-                .build();
-
-        var bytes = IOUtils.toByteArray(stream);
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
-
-        return request;
-
-    }
-
-
    @Test
    public void sponsorCompanyTest() throws IOException {

@@ -1517,7 +1210,8 @@ public class RedactionIntegrationTest {
            fileOutputStream.write(annotateResponse.getDocument());
        }

-        var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions);
+        var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions)
+                .getManualRedactions();
        surroundingTextResult.getEntriesToAdd().forEach(addEntry -> {
            assertThat(addEntry.getTextAfter()).isNotEmpty();
        });
@@ -1525,6 +1219,117 @@ public class RedactionIntegrationTest {
    }


+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(RECOMMENDATION_AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/recommendation_CBI_author.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(RECOMMENDATION_ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/recommendation_CBI_address.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FALSE_POSITIVE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/false_positive.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+    }
+
+
    private static String loadFromClassPath(String path) {

        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
@@ -1561,4 +1366,243 @@ public class RedactionIntegrationTest {

    }

-}
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
+        typeColorMap.put(ADDRESS, "#ffe187");
+        typeColorMap.put(AUTHOR, "#ffe187");
+        typeColorMap.put(SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
+        typeColorMap.put(PII, "#66ccff");
+        typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
+        typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
+        typeColorMap.put(FALSE_POSITIVE, "#ffffff");
+        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION, "#32a852");
+
+        hintTypeMap.put(VERTEBRATE, true);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
+        hintTypeMap.put(SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
+        hintTypeMap.put(PII, false);
+        hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
+        hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
+        hintTypeMap.put(FALSE_POSITIVE, true);
+        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);
+        hintTypeMap.put(IMPORTED_REDACTION, false);
+
+        caseInSensitiveMap.put(VERTEBRATE, true);
+        caseInSensitiveMap.put(ADDRESS, false);
+        caseInSensitiveMap.put(AUTHOR, false);
+        caseInSensitiveMap.put(SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
+        caseInSensitiveMap.put(PII, false);
+        caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
+        caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
+        caseInSensitiveMap.put(FALSE_POSITIVE, false);
+        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+
+        recommendationTypeMap.put(VERTEBRATE, false);
+        recommendationTypeMap.put(ADDRESS, false);
+        recommendationTypeMap.put(AUTHOR, false);
+        recommendationTypeMap.put(SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY, false);
+        recommendationTypeMap.put(MUST_REDACT, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
+        recommendationTypeMap.put(TEST_METHOD, false);
+        recommendationTypeMap.put(PII, false);
+        recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
+        recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
+        recommendationTypeMap.put(FALSE_POSITIVE, false);
+        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+
+        rankTypeMap.put(FALSE_POSITIVE, 160);
+        rankTypeMap.put(PURITY, 155);
+        rankTypeMap.put(PII, 150);
+        rankTypeMap.put(ADDRESS, 140);
+        rankTypeMap.put(AUTHOR, 130);
+        rankTypeMap.put(SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE, 110);
+        rankTypeMap.put(MUST_REDACT, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
+        rankTypeMap.put(TEST_METHOD, 60);
+        rankTypeMap.put(HINT_ONLY, 50);
+        rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
+        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
+        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
+        rankTypeMap.put(IMPORTED_REDACTION, 200);
+
+        colors.setDefaultColor("#acfc00");
+        colors.setNotRedacted("#cccccc");
+        colors.setRequestAdd("#04b093");
+        colors.setRequestRemove("#04b093");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(DictionaryEntry.builder()
+                    .value(entry)
+                    .version(reanlysisVersions.getOrDefault(entry, 0L))
+                    .deleted(deleted.contains(entry))
+                    .build());
+        });
+        return dictionaryEntries;
+    }
+
+
+    @Test
+    public void testImportedRedactions() throws IOException {
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+        ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.pdf");
+        ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.json");
+
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(InputStream stream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        var bytes = IOUtils.toByteArray(stream);
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
+
+        return request;
+
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(String file) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+
+        return prepareStorage(pdfFileResource.getInputStream());
+    }
+
+}
@@ -0,0 +1 @@
+{"importedRedactions":{"1":[{"id":"755273b5ddc6933eff7f6b6e055c803f","positions":[{"topLeft":{"x":275.0,"y":520.504},"width":55.675,"height":15.582,"page":1}]},{"id":"5e517d2c890272fc140578a26971b459","positions":[{"topLeft":{"x":164.0,"y":316.492},"width":50.7,"height":15.658,"page":1},{"topLeft":{"x":127.0,"y":302.613},"width":119.675,"height":14.902,"page":1},{"topLeft":{"x":81.0,"y":289.622},"width":210.025,"height":14.846,"page":1},{"topLeft":{"x":136.0,"y":275.912},"width":94.2,"height":13.033,"page":1},{"topLeft":{"x":158.0,"y":261.402},"width":52.793,"height":16.218,"page":1},{"topLeft":{"x":177.0,"y":247.413},"width":15.507,"height":16.148,"page":1}]}]}}
@@ -0,0 +1,12 @@
+{
+  "result": {
+    "1": [
+      {
+        "value": "Mannheim",
+        "startOffset": 0,
+        "endOffset": 8,
+        "type": "CITY"
+      }
+    ]
+  }
+}
Author	SHA1	Message	Date
Timo Bejan	de92117b35	added imported flag	2022-02-11 11:33:34 +02:00
Philipp Schramm	143f4cfb0d	Pull request #341 : RED-3350 Communication with entity recognition via queues Merge in RED/redaction-service from feature/RED-3350 to master * commit '68727178e79224b7ca7f9f293e48ca21b4ce2f91': RED-3350 Communication with entity recognition via queues	2022-02-09 15:24:47 +01:00
Philipp Schramm	68727178e7	RED-3350 Communication with entity recognition via queues	2022-02-09 15:19:55 +01:00
Dominique Eiflaender	90fe282313	Pull request #339 : RED-3327: Ignore recommendations if touches other type Merge in RED/redaction-service from RED-3327-master to master * commit '8689125ed9b09f73781d3babfd65537c0b1f4afe': RED-3327: Ignore recommendations if touches other type	2022-02-01 11:19:26 +01:00
deiflaender	8689125ed9	RED-3327: Ignore recommendations if touches other type	2022-02-01 11:06:32 +01:00
Dominique Eiflaender	e68c6798be	Pull request #337 : RED-3243: Improved imported redactions logic Merge in RED/redaction-service from RED-3243-8 to master * commit '58b1db66cc2cc67b3b4e26295478824e5052d6a3': RED-3243: Improved imported redactions logic	2022-01-31 14:47:54 +01:00
deiflaender	58b1db66cc	RED-3243: Improved imported redactions logic	2022-01-31 14:42:49 +01:00
deiflaender	0072d38d68	Use base image to speedup build	2022-01-31 12:05:10 +01:00
deiflaender	25a409d3a5	RED-3243: Added ImportedRedaction model classes	2022-01-31 11:11:32 +01:00
Timo Bejan	ca1da20ba3	Pull request #335 : immutable field fix Merge in RED/redaction-service from RED-3242 to master * commit '3c691a1afc9ce67845aad5d15bbc99a0113f6a6b': immutable field fix	2022-01-28 10:57:04 +01:00
Timo Bejan	3c691a1afc	immutable field fix	2022-01-28 11:29:43 +02:00
Timo Bejan	6813fbc3cb	Pull request #333 : since no more manual changes are fields of redactionLog, we can safely always merge, even legacy ones Merge in RED/redaction-service from RED-3242 to master * commit 'c9a75152fb9b4e02b3421dd16ae54056889b0aa9': since no more manual changes are fields of redactionLog, we can safely always merge, even legacy ones	2022-01-27 15:59:39 +01:00
Dominique Eiflaender	a8a6fef07a	Pull request #334 : RED-3239: Protoyped logic for imported redactions Merge in RED/redaction-service from RED-3239 to master * commit '26661fa87beeb20b32f2be5a8b1b025e8d2d29fd': RED-3239: Protoyped logic for imported redactions	2022-01-27 15:31:16 +01:00
deiflaender	26661fa87b	RED-3239: Protoyped logic for imported redactions	2022-01-27 15:21:33 +01:00
Timo Bejan	c9a75152fb	since no more manual changes are fields of redactionLog, we can safely always merge, even legacy ones	2022-01-27 15:30:43 +02:00
Timo Bejan	0da2a1ab26	Pull request #332 : RED-3242 - excluded from automatic analysis - reworked redactionLog architecture Merge in RED/redaction-service from RED-3242 to master * commit 'd92a723fea21280d7bf482c3d74c7b177dbaa030': RED-3242 - excluded from automatic analysis - reworked redactionLog architecture	2022-01-27 14:06:35 +01:00
Timo Bejan	d92a723fea	RED-3242 - excluded from automatic analysis - reworked redactionLog architecture	2022-01-27 14:58:14 +02:00
Dominique Eiflaender	3109e3e4f0	Pull request #331 : RED-3133: Ignore min length for values from ai Merge in RED/redaction-service from RED-3133-Fix4 to master * commit 'a1544588e8a3cfa0479d5d31d82463c5434674a1': RED-3133: Ignore min length for values from ai	2022-01-26 16:32:23 +01:00
deiflaender	a1544588e8	RED-3133: Ignore min length for values from ai	2022-01-26 16:18:41 +01:00
Dominique Eiflaender	8ba2bd776d	Pull request #330 : RED-3133: Fixed wrong ai entries again Merge in RED/redaction-service from RED-3133-Fix2 to master * commit '8ac7c309d981c8899eca40872a49198284d15944': RED-3133: Fixed wrong ai entries again	2022-01-26 15:42:13 +01:00
deiflaender	8ac7c309d9	RED-3133: Fixed wrong ai entries again	2022-01-26 15:31:13 +01:00
Dominique Eiflaender	b8f7be28e7	Pull request #329 : RED-3133: Fixed wrong ai entries Merge in RED/redaction-service from RED-3133-Fix to master * commit '921ae1839cb424a4d82afd88401fddc9cf855f8c': RED-3133: Fixed wrong ai entries	2022-01-26 15:09:04 +01:00
deiflaender	921ae1839c	RED-3133: Fixed wrong ai entries	2022-01-26 14:43:54 +01:00
				`@@ -0,0 +1 @@`
				{"importedRedactions":{"1":[{"id":"755273b5ddc6933eff7f6b6e055c803f","positions":[{"topLeft":{"x":275.0,"y":520.504},"width":55.675,"height":15.582,"page":1}]},{"id":"5e517d2c890272fc140578a26971b459","positions":[{"topLeft":{"x":164.0,"y":316.492},"width":50.7,"height":15.658,"page":1},{"topLeft":{"x":127.0,"y":302.613},"width":119.675,"height":14.902,"page":1},{"topLeft":{"x":81.0,"y":289.622},"width":210.025,"height":14.846,"page":1},{"topLeft":{"x":136.0,"y":275.912},"width":94.2,"height":13.033,"page":1},{"topLeft":{"x":158.0,"y":261.402},"width":52.793,"height":16.218,"page":1},{"topLeft":{"x":177.0,"y":247.413},"width":15.507,"height":16.148,"page":1}]}]}}