Fixed endless processing

Ignore failing Ruling expansion
Fixed text parsing orientation problem
2021-06-02 13:49:48 +02:00 · 2021-06-02 13:00:51 +02:00 · 2021-06-02 11:19:34 +02:00 · 2021-06-01 15:41:14 +02:00 · 2021-06-01 15:40:41 +02:00 · 2021-05-12 12:23:03 +02:00
49 changed files with 774 additions and 234 deletions
@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.3.0</version>
+                <version>1.3.1</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -21,7 +21,9 @@ public class AnalyzeResult {
    private boolean hasImages;
    private boolean hasUpdates;
    private long dictionaryVersion;
+    private long dossierDictionaryVersion;
    private long rulesVersion;

+
 }

@@ -0,0 +1,5 @@
+package com.iqser.red.service.redaction.v1.model;
+
+public enum ChangeType {
+    ADDED, REMOVED
+}
@@ -27,4 +27,6 @@ public class ManualRedactionEntry {
    private String section;
    private int sectionNumber;

+    private boolean addToDossierDictionary;
+
 }
@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class RedactionChangeLog {
+
+    private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
+
+    private long dictionaryVersion = -1;
+    private long rulesVersion = -1;
+
+    private String ruleSetId;
+
+}
@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RedactionChangeLogEntry {
+
+    private String id;
+    private String type;
+    private String value;
+    private String reason;
+    private int matchedRule;
+    private String legalBasis;
+    private boolean redacted;
+    private boolean isHint;
+    private boolean isRecommendation;
+    private String section;
+    private float[] color;
+
+    @Builder.Default
+    private List<Rectangle> positions = new ArrayList<>();
+    private int sectionNumber;
+    private boolean manual;
+    private Status status;
+    private ManualRedactionType manualRedactionType;
+    private boolean isDictionaryEntry;
+
+    private String textBefore;
+    private String textAfter;
+
+    @Builder.Default
+    private List<Comment> comments = new ArrayList<>();
+
+    private ChangeType changeType;
+
+    private boolean isDossierDictionaryEntry;
+
+}
@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.model;

-import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

 import java.util.List;

@Data
-@AllArgsConstructor
@NoArgsConstructor
 public class RedactionLog {

@@ -17,15 +15,17 @@ public class RedactionLog {
    private long rulesVersion = -1;

    private String ruleSetId;
-    private String filename;
+
+    private long dossierDictionaryVersion = -1;


-    public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId) {
+    public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId, long dossierDictionaryVersion) {

        this.redactionLogEntry = redactionLogEntry;
        this.dictionaryVersion = dictionaryVersion;
        this.rulesVersion = rulesVersion;
        this.ruleSetId = ruleSetId;
+        this.dossierDictionaryVersion = dossierDictionaryVersion;
    }


@@ -45,4 +45,6 @@ public class RedactionLogEntry {

    private boolean isImage;

+    private boolean isDossierDictionaryEntry;
+
 }
@@ -13,11 +13,6 @@ public interface RedactionResource {
    String RULE_SET_PARAMETER_NAME = "ruleSetId";
    String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";

-    @PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
-
-    @PostMapping(value = "/reanalyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    AnalyzeResult reanalyze(@RequestBody AnalyzeRequest renalyzeRequest);

    @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
@@ -24,12 +24,18 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>configuration-service-api-v1</artifactId>
-            <version>2.5.0</version>
+            <version>2.5.6</version>
        </dependency>
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>file-management-service-api-v1</artifactId>
-            <version>2.6.7</version>
+            <version>2.7.4</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;

 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import lombok.Data;
@@ -31,7 +32,7 @@ public class Document {

    private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
    private SectionGrid sectionGrid = new SectionGrid();
-    private long dictionaryVersion;
+    private DictionaryVersion dictionaryVersion;
    private long rulesVersion;

    private List<SectionText> sectionText = new ArrayList<>();
@@ -0,0 +1,6 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+public enum Orientation {
+
+    NONE, LEFT, RIGHT
+}
@@ -32,6 +32,7 @@ public class TextBlock extends AbstractTextContainer {

    private String classification;

+
    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
        this.minX = minX;
        this.maxX = maxX;
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;

 import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -11,16 +12,21 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
 import org.springframework.stereotype.Service;

 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;

@Service
@SuppressWarnings("all")
 public class BlockificationService {

-    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    static final float THRESHOLD = 1f;
+
+    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
+                         List<Ruling> verticalRulingLines) {

        List<TextPositionSequence> chunkWords = new ArrayList<>();
        List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
@@ -28,21 +34,46 @@ public class BlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;

+        boolean wasSplitted = false;
+        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

            boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
            boolean startFromTop = word.getY1() > maxY + word.getHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
+            boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
+            boolean splittedByRuling = word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines) || word
+                    .getRotation() == 0 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines) || word
+                    .getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
+                    .getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);

-            if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
-                    .getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
-                    .getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
-                    .getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
-                    .getX1(), word.getY2(), verticalRulingLines))) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
+
+                Orientation prevOrientation = null;
+                if(!chunkBlockList1.isEmpty()) {
+                    prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
+                }

                TextBlock cb1 = buildTextBlock(chunkWords);
                chunkBlockList1.add(cb1);
                chunkWords = new ArrayList<>();

+                if (splitByX && !splittedByRuling) {
+                    wasSplitted = true;
+                    cb1.setOrientation(Orientation.LEFT);
+                    splitX1 = word.getX1();
+                } else
+
+                if (newLineAfterSplit && !splittedByRuling) {
+                    wasSplitted = false;
+                    cb1.setOrientation(Orientation.RIGHT);
+                    splitX1 = null;
+                } else
+
+                if(prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !splittedByRuling)){
+                    cb1.setOrientation(Orientation.LEFT);
+                }
+
                minX = 1000;
                maxX = 0;
                minY = 1000;
@@ -72,9 +103,62 @@ public class BlockificationService {
            chunkBlockList1.add(cb1);
        }

+        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+
+        TextBlock previousLeft = null;
+        TextBlock previousRight = null;
+        while (itty.hasNext()) {
+            TextBlock block = (TextBlock) itty.next();
+
+            if(previousLeft != null && block.getOrientation().equals(Orientation.LEFT)){
+                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()){
+                    previousLeft.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if(previousRight != null && block.getOrientation().equals(Orientation.RIGHT)){
+                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()){
+                    previousRight.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if (block.getOrientation().equals(Orientation.LEFT)) {
+                previousLeft = block;
+            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+                previousRight = block;
+            }
+        }
+
+
+        itty = chunkBlockList1.iterator();
+        TextBlock previous = null;
+        while (itty.hasNext()) {
+            TextBlock block = (TextBlock) itty.next();
+
+            if(previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous
+                    .getMaxY())||
+                    previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous
+                    .getMaxY())){
+                previous.add(block);
+                itty.remove();
+                continue;
+            }
+
+            previous = block;
+        }
+
+
        return new Page(chunkBlockList1);
    }

+    private boolean equalsWithThreshold(float f1, float f2){
+        return Math.abs(f1 - f2) < THRESHOLD;
+    }
+

    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {

@@ -117,7 +201,8 @@ public class BlockificationService {
    }


-    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
+    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
+                                       List<Ruling> rulingLines) {

        for (Ruling ruling : rulingLines) {
            if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
@@ -128,7 +213,8 @@ public class BlockificationService {
    }


-    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
+    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
+                                            boolean landscape) {

        float minX = 10000;
        float maxX = -100;
@@ -0,0 +1,9 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+
+import com.iqser.red.service.file.management.v1.api.resources.FileStatusProcessingUpdateResource;
+import org.springframework.cloud.openfeign.FeignClient;
+
+@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${file-management-service.url}")
+public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
+}
@@ -3,6 +3,6 @@ package com.iqser.red.service.redaction.v1.server.client;
 import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
 import org.springframework.cloud.openfeign.FeignClient;

-@FeignClient(name = RulesResource.SERVICE_NAME, url = "${configuration-service.url}")
+@FeignClient(name = "RulesResource", url = "${configuration-service.url}")
 public interface RulesClient extends RulesResource {
-}
+}
@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.controller;

 import com.iqser.red.service.file.management.v1.api.model.FileType;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
@@ -9,7 +12,6 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -35,21 +37,9 @@ public class RedactionController implements RedactionResource {
    private final DroolsExecutionService droolsExecutionService;
    private final DictionaryService dictionaryService;
    private final AnnotationService annotationService;
-    private final ReanalyzeService reanalyzeService;
    private final PdfSegmentationService pdfSegmentationService;
    private final RedactionStorageService redactionStorageService;

-    @Override
-    public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
-        return reanalyzeService.analyze(analyzeRequest);
-
-    }
-
-    @Override
-    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
-        return reanalyzeService.reanalyze(analyzeRequest);
-    }
-

    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {

@@ -60,7 +50,7 @@ public class RedactionController implements RedactionResource {
        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
            pdDocument.setAllSecurityToBeRemoved(true);

-            dictionaryService.updateDictionary(redactionLog.getRuleSetId());
+            dictionaryService.updateDictionary(redactionLog.getRuleSetId(), annotateRequest.getProjectId());
            annotationService.annotate(pdDocument, redactionLog, sectionsGrid);

            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
@@ -46,6 +46,17 @@ public class PDFAreaTextStripper extends PDFTextStripperByArea {
                startIndex = i;
            }

+
+            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i;
+            }
+
            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
                    .getUnicode()
                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
@@ -300,6 +300,18 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                startIndex = i;
            }

+
+            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                }
+                startIndex = i;
+            }
+
+
            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
                    .getUnicode()
                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
@@ -21,7 +21,7 @@ public class MessagingConfiguration {

        return QueueBuilder.durable(REDACTION_QUEUE)
                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_QUEUE)
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
                .maxPriority(2)
                .build();
    }
@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.queue;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -20,27 +22,33 @@ public class RedactionMessageReceiver {

    private final ObjectMapper objectMapper;
    private final ReanalyzeService reanalyzeService;
+    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;

    @RabbitHandler
    @RabbitListener(queues = REDACTION_QUEUE)
    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+
        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
        log.info("Processing analyze request: {}", analyzeRequest);
+        AnalyzeResult result;
        if (analyzeRequest.isReanalyseOnlyIfPossible()) {
-            reanalyzeService.reanalyze(analyzeRequest);
+            result = reanalyzeService.reanalyze(analyzeRequest);
        } else {
-            reanalyzeService.analyze(analyzeRequest);
+            result = reanalyzeService.analyze(analyzeRequest);
        }
        log.info("Successfully analyzed {}", analyzeRequest);
-        // TODO respond to file-api
+
+        fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), result);
    }

    @RabbitHandler
    @RabbitListener(queues = REDACTION_DQL)
    public void receiveAnalyzeRequestDQL(String in) throws JsonProcessingException {
+
        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
        log.info("Failed to process analyze request: {}", analyzeRequest);
-        // TODO respond to file-api
+
+        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getProjectId(), analyzeRequest.getFileId());
    }

 }
@@ -18,13 +18,13 @@ public class Dictionary {
    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();

    @Getter
-    private long version;
+    private DictionaryVersion version;


-    public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion) {
+    public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
        this.dictionaryModels = dictionaryModels;
        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
-        this.version = dictionaryVersion;
+        this.version = version;
    }


@@ -10,6 +10,6 @@ import java.util.Set;
 public class DictionaryIncrement {

    private Set<DictionaryIncrementValue> values;
-    private long dictionaryVersion;
+    private DictionaryVersion dictionaryVersion;

 }
@@ -21,6 +21,7 @@ public class DictionaryModel implements Serializable {
    private boolean recommendation;
    private Set<DictionaryEntry> entries;
    private Set<String> localEntries;
+    private boolean isDossierDictionary;

    public Set<String> getValues(boolean local) {
        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DictionaryVersion {
+
+    long rulesetVersion;
+    long dossierVersion;
+}
@@ -37,8 +37,10 @@ public class Entity {
    private String textBefore;
    private String textAfter;

+    private boolean isDossierDictionaryEntry;

-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end) {
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end, boolean isDossierDictionaryEntry) {

        this.word = word;
        this.type = type;
@@ -54,10 +56,11 @@ public class Entity {
        this.textAfter = textAfter;
        this.start = start;
        this.end = end;
+        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
    }


-    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry) {

        this.word = word;
        this.type = type;
@@ -66,6 +69,7 @@ public class Entity {
        this.headline = headline;
        this.sectionNumber = sectionNumber;
        this.isDictionaryEntry = isDictionaryEntry;
+        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
    }

 }
@@ -411,7 +411,7 @@ public class Section {
        String text = caseInsensitive ? searchText.toLowerCase() : searchText;
        String searchValue = caseInsensitive ? value.toLowerCase() : value;

-        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true);
+        Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true, false);

        found.forEach(entity -> {
            if (redacted) {
@@ -437,7 +437,7 @@ public class Section {
        } else {
            String word = value.toString();

-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
+            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false);
            entity.setRedaction(redact);
            entity.setMatchedRule(ruleNumber);
            entity.setRedactionReason(reason);
@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.file.management.v1.api.model.RedactionChangeLog;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import org.springframework.stereotype.Service;
@@ -42,6 +42,7 @@ public class AnalyzeResponseService {
                .hasUpdates(hasUpdates)
                .rulesVersion(redactionLog.getRulesVersion())
                .dictionaryVersion(redactionLog.getDictionaryVersion())
+                .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
                .build();
    }
 }
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER;
+
 import com.iqser.red.service.configuration.v1.api.model.Colors;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -10,6 +12,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncre
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -29,52 +33,68 @@ public class DictionaryService {
    private final DictionaryClient dictionaryClient;

    private final Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();


-    public long updateDictionary(String ruleSetId) {
+    public DictionaryVersion updateDictionary(String ruleSetId, String dossierId) {

-        long version = dictionaryClient.getVersion(ruleSetId);
-
-        var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
-
-        if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(ruleSetId, version);
+        long rulesetDictionaryVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER);
+        var rulesetDictionary = dictionariesByRuleSets.get(ruleSetId);
+        if (rulesetDictionary == null || rulesetDictionaryVersion > rulesetDictionary.getDictionaryVersion()) {
+            updateDictionaryEntry(ruleSetId, rulesetDictionaryVersion, GLOBAL_DOSSIER);
        }

-        return version;
+        long dossierDictionaryVersion = dictionaryClient.getVersion(ruleSetId, dossierId);
+        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
+            updateDictionaryEntry(ruleSetId, dossierDictionaryVersion, dossierId);
+        }
+
+        return DictionaryVersion.builder().rulesetVersion(rulesetDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
    }


-    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, long fromVersion) {
+    public DictionaryIncrement getDictionaryIncrements(String ruleSetId, DictionaryVersion fromVersion, String dossierId) {

-        long version = updateDictionary(ruleSetId);
+        DictionaryVersion version = updateDictionary(ruleSetId, dossierId);

        Set<DictionaryIncrementValue> newValues = new HashSet<>();
        List<DictionaryModel> dictionaryModels = dictionariesByRuleSets.get(ruleSetId).getDictionary();
        dictionaryModels.forEach(dictionaryModel -> {
            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
-                if (dictionaryEntry.getVersion() > fromVersion) {
+                if (dictionaryEntry.getVersion() > fromVersion.getRulesetVersion()) {
                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
                }
            });
        });

+        if(dictionariesByDossier.containsKey(dossierId)) {
+            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+            dictionaryModels.forEach(dictionaryModel -> {
+                dictionaryModel.getEntries().forEach(dictionaryEntry -> {
+                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                    }
+                });
+            });
+        }
+
        return new DictionaryIncrement(newValues, version);
    }


-    private void updateDictionaryEntry(String ruleSetId, long version) {
+    private void updateDictionaryEntry(String ruleSetId, long version, String dossierId) {

        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();

-            TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId);
+            TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId, dossierId);
            if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {

                List<DictionaryModel> dictionary = typeResponse.getTypes()
                        .stream()
                        .map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                .isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
+                                .isHint(), t.isRecommendation(), convertEntries(t, dossierId), new HashSet<>(),dossierId.equals(GLOBAL_DOSSIER) ? false : true))
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());

@@ -90,7 +110,11 @@ public class DictionaryService {
                dictionaryRepresentation.setDictionaryVersion(version);
                dictionaryRepresentation.setDictionary(dictionary);

-                dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
+                if(dossierId.equals(GLOBAL_DOSSIER)) {
+                    dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
+                } else {
+                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                }
            }
        } catch (FeignException e) {
            log.warn("Got some unknown feignException", e);
@@ -103,19 +127,19 @@ public class DictionaryService {

        dictionary.getDictionaryModels().forEach(dm -> {
            if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
-                dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
-                long externalVersion = dictionaryClient.getVersion(ruleSetId);
-                if (externalVersion == dictionary.getVersion() + 1) {
-                    dictionary.setVersion(externalVersion);
+                dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER);
+                long externalVersion = dictionaryClient.getVersion(ruleSetId, GLOBAL_DOSSIER);
+                if (externalVersion == dictionary.getVersion().getRulesetVersion() + 1) {
+                    dictionary.getVersion().setRulesetVersion(externalVersion);
                }
            }
        });
    }


-    private Set<DictionaryEntry> convertEntries(TypeResult t) {
+    private Set<DictionaryEntry> convertEntries(TypeResult t, String dossierId) {

-        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
+        Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId(), dossierId)
                .getEntries());

        if (t.isCaseInsensitive()) {
@@ -172,17 +196,26 @@ public class DictionaryService {
    }


-    public Dictionary getDeepCopyDictionary(String ruleSetId) {
+    public Dictionary getDeepCopyDictionary(String ruleSetId, String dossierId) {

        List<DictionaryModel> copy = new ArrayList<>();

-        var representation = dictionariesByRuleSets.get(ruleSetId);
-        var dictionary = dictionariesByRuleSets.get(ruleSetId).getDictionary();
-        dictionary.forEach(dm -> {
+        var rulesetRepresentation = dictionariesByRuleSets.get(ruleSetId);
+        rulesetRepresentation.getDictionary().forEach(dm -> {
            copy.add(SerializationUtils.clone(dm));
        });

-        return new Dictionary(copy, representation.getDictionaryVersion());
+        //TODO merge dictionaries if they have same names
+        long dossierDictionaryVersion = -1;
+        if(dictionariesByDossier.containsKey(dossierId)) {
+            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+            dossierRepresentation.getDictionary().forEach(dm -> {
+                copy.add(SerializationUtils.clone(dm));
+            });
+            dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
+        }
+
+        return new Dictionary(copy, DictionaryVersion.builder().rulesetVersion(rulesetRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
    }


@@ -33,13 +33,13 @@ public class EntityRedactionService {
    private final SurroundingWordsService surroundingWordsService;


-    public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
+    public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions, String dossierId) {

-        dictionaryService.updateDictionary(ruleSetId);
+        dictionaryService.updateDictionary(ruleSetId, dossierId);
        KieContainer container = droolsExecutionService.updateRules(ruleSetId);
        long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId, dossierId);

        Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));

@@ -71,7 +71,7 @@ public class EntityRedactionService {
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                .getStart(), entity.getEnd()));
+                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
            }
        }

@@ -371,9 +371,9 @@ public class EntityRedactionService {
        String lowercaseInputString = searchableString.toLowerCase();
        for (DictionaryModel model : dictionary.getDictionaryModels()) {
            if (model.isCaseInsensitive()) {
-                found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
            } else {
-                found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
            }
        }

@@ -11,9 +11,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -37,14 +39,17 @@ public class ReanalyzeService {
    private final RedactionChangeLogService redactionChangeLogService;
    private final AnalyzeResponseService analyzeResponseService;

+
    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
        long startTime = System.currentTimeMillis();

        var pageCount = 0;
        Document classifiedDoc;

        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.ORIGIN));
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
+                    .getProjectId(), analyzeRequest.getFileId(), FileType.ORIGIN));
            classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
            pageCount = classifiedDoc.getPages().size();
        } catch (Exception e) {
@@ -52,24 +57,28 @@ public class ReanalyzeService {
        }
        log.info("Document structure analysis successful, starting redaction analysis...");

-        entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions());
+        entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions(), analyzeRequest
+                .getProjectId());
        redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
                .getRuleSetId());

        log.info("Redaction analysis successful...");

+        var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion()
+                .getRulesetVersion(), classifiedDoc.getRulesVersion(), analyzeRequest.getRuleSetId(), classifiedDoc.getDictionaryVersion()
+                .getDossierVersion());

-        var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
-                .getRulesVersion(), analyzeRequest.getRuleSetId());
-
-        log.info("Analyzed with rules {} and dictionary {} for ruleSet: {}", classifiedDoc.getRulesVersion(), classifiedDoc.getDictionaryVersion(), analyzeRequest.getRuleSetId());
+        log.info("Analyzed with rules {} and dictionary {} for ruleSet: {}", classifiedDoc.getRulesVersion(), classifiedDoc
+                .getDictionaryVersion(), analyzeRequest.getRuleSetId());

        // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
        var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog);
        // store redactionLog
        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
-        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc.getSectionText()));
-        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
+        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
+                .getSectionText()));
+        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
+                .getSectionGrid());

        long duration = System.currentTimeMillis() - startTime;
        return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
@@ -78,17 +87,19 @@ public class ReanalyzeService {

    @SneakyThrows
    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
+
        long startTime = System.currentTimeMillis();

        var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId());
        var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId());

        // not yet ready for reanalysis
-        if (text.getNumberOfPages() == 0) {
+        if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
            return analyze(analyzeRequest);
        }

-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getRuleSetId(), redactionLog.getDictionaryVersion());
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getRuleSetId(), new DictionaryVersion(redactionLog
+                .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getProjectId());

        Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
        Map<String, List<Comment>> comments = null;
@@ -146,12 +157,11 @@ public class ReanalyzeService {
            }
        }

-
        //--

        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getRuleSetId());

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getRuleSetId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getRuleSetId(), analyzeRequest.getProjectId());

        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
        for (SectionText reanalysisSection : reanalysisSections) {
@@ -183,8 +193,7 @@ public class ReanalyzeService {
        Set<Entity> entities = new HashSet<>();
        Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
-            Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair
-                    .getSection());
+            Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
            entities.addAll(analysedRowSection.getEntities());
            EntitySearchUtils.removeEntitiesContainedInLarger(entities);

@@ -207,43 +216,48 @@ public class ReanalyzeService {
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                .getStart(), entity.getEnd()));
+                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
            }
        }

        List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
        for (int page = 1; page <= text.getNumberOfPages(); page++) {
            if (entitiesPerPage.get(page) != null) {
-                newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest
-                        .getManualRedactions(), page, analyzeRequest.getRuleSetId()));
+                newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
+                        .getRuleSetId()));
            }

            if (imagesPerPage.get(page) != null) {
-                newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest
-                        .getManualRedactions(), page, analyzeRequest.getRuleSetId()));
+                newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
+                        .getRuleSetId()));
            }

            newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
                    .getRuleSetId()));
        }

-
-        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.isImage() || entry.getSectionNumber() == 0 && !entry.isImage());
+        redactionLog.getRedactionLogEntry()
+                .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
        redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
-        return finalizeAnalysis
-                (analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);

    }

-    private AnalyzeResult finalizeAnalysis(@RequestBody AnalyzeRequest analyzeRequest, long startTime, RedactionLog redactionLog, Text text, DictionaryIncrement dictionaryIncrement) {
-        redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
+
+    private AnalyzeResult finalizeAnalysis(@RequestBody AnalyzeRequest analyzeRequest, long startTime,
+                                           RedactionLog redactionLog, Text text,
+                                           DictionaryIncrement dictionaryIncrement) {
+
+        redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getRulesetVersion());
+        redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion());

        var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), redactionLog);
        redactionStorageService.storeObject(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);

        long duration = System.currentTimeMillis() - startTime;

-        return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, text.getNumberOfPages(), redactionLog, changeLog);
+        return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getProjectId(), analyzeRequest.getFileId(), duration, text
+                .getNumberOfPages(), redactionLog, changeLog);
    }


@@ -1,10 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.file.management.v1.api.model.ChangeType;
 import com.iqser.red.service.file.management.v1.api.model.FileType;
-import com.iqser.red.service.file.management.v1.api.model.RedactionChangeLog;
-import com.iqser.red.service.file.management.v1.api.model.RedactionChangeLogEntry;
+import com.iqser.red.service.redaction.v1.model.ChangeType;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
+import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -21,15 +20,14 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
 public class RedactionChangeLogService {

-    private final RedactionStorageService storageStorageService;
-    private final ObjectMapper objectMapper;
+    private final RedactionStorageService redactionStorageService;

    public RedactionChangeLog createAndStoreChangeLog(String projectId, String fileId, RedactionLog currentRedactionLog) {

        try {
-            RedactionLog previousRedactionLog = storageStorageService.getRedactionLog(projectId, fileId);
+            RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(projectId, fileId);
            var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
-            storageStorageService.storeObject(projectId, fileId, FileType.REDACTION_CHANGELOG, objectMapper.writeValueAsBytes(changeLog));
+            redactionStorageService.storeObject(projectId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
            return changeLog;
        } catch (Exception e) {
            log.debug("Previous redaction log not available");
@@ -60,7 +58,7 @@ public class RedactionChangeLogService {
                .collect(Collectors.toList()));

        return new RedactionChangeLog(changeLogEntries, currentRedactionLog.getDictionaryVersion(), currentRedactionLog.getRulesVersion(), currentRedactionLog
-                .getRuleSetId(), currentRedactionLog.getFilename());
+                .getRuleSetId());
    }


@@ -88,6 +86,7 @@ public class RedactionChangeLogService {
                .textAfter(entry.getTextAfter())
                .comments(entry.getComments())
                .changeType(changeType)
+                .isDossierDictionaryEntry(entry.isDossierDictionaryEntry())
                .build();
    }

@@ -355,6 +355,7 @@ public class RedactionLogCreatorService {
                .status(manualRedactionEntry.getStatus())
                .manualRedactionType(ManualRedactionType.ADD)
                .isDictionaryEntry(false)
+                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
                .build();
    }

@@ -378,6 +379,7 @@ public class RedactionLogCreatorService {
                .textBefore(entity.getTextBefore())
                .startOffset(entity.getStart())
                .endOffset(entity.getEnd())
+                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
                .build();
    }

@@ -49,7 +49,7 @@ public class EntitySearchUtils {


    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean local) {
+                            boolean local, boolean isDossierDictionary) {

        Set<Entity> found = new HashSet<>();

@@ -69,7 +69,7 @@ public class EntitySearchUtils {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local, isDossierDictionary));
                }
            } while (startIndex > -1);
        }
@@ -8,6 +8,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi
 import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
@@ -22,6 +23,9 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.springframework.stereotype.Service;

+import java.awt.Graphics;
+import java.awt.geom.Rectangle2D;
+import java.awt.image.BufferedImage;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -92,7 +96,8 @@ public class PdfSegmentationService {
                page.setRotation(rotation);
                page.setLandscape(isLandscape || isRotated);
                page.setPageNumber(pageNumber);
-                page.setImages(stripper.getImages());
+                List<PdfImage> mergedList = processImages(stripper.getImages());
+                page.setImages(mergedList);

                tableExtractionService.extractTables(cleanRulings, page);
                buildPageStatistics(page);
@@ -105,6 +110,7 @@ public class PdfSegmentationService {

                pages.add(page);

+
            }

            document.setPages(pages);
@@ -142,6 +148,130 @@ public class PdfSegmentationService {
        return newPDDocument;
    }

+    //merge images, if they are separated during pdf import, return new list of Pdfimages
+    private List<PdfImage> processImages(List<PdfImage> imageList) {
+        if (imageList.size() > 1) {
+            List<PdfImage> mergedList = new ArrayList<>();
+            int countElementsInList = 0;
+            boolean beginImage = true;
+
+            // a List of Boolean, true = candidate for merging, false = no merging
+            List<Boolean> candidatesList = getCandidatesList(imageList);
+
+            // loop through list, if there are candidates for merging (true), merge images and add it to mergedList
+            for (int i = 0; i < candidatesList.size(); i++) {
+                if (candidatesList.get(i)) {
+                    if (beginImage) {
+                        //begin of image, merge two parts of imageList
+                        PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1));
+                        // image merge successful
+                        if (mergedImage != null) {
+                            mergedList.add(mergedImage);
+                            countElementsInList++;
+                        }
+                    } else {
+                        //middle of an image, merge current piece auf mergedList with image of imageList
+                        PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1));
+                        // image merge successful
+                        if (mergedImage != null) {
+                            mergedList.set(countElementsInList - 1, mergedImage);
+                        }
+                    }
+                    beginImage = false;
+                } else {
+                    // if the last candidate is false, then both images i and i+1 must be added
+                    if (i == candidatesList.size() - 1) {
+                        if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) {
+                            mergedList.add(imageList.get(i + 1));
+                        } else {
+                            mergedList.add(imageList.get(i));
+                            mergedList.add(imageList.get(i + 1));
+                        }
+                    } else {
+                        //first image is not splitted, add i to resultlist
+                        if (beginImage) {
+                            mergedList.add(imageList.get(i));
+                            countElementsInList++;
+                        } else {
+                            // i is the end of an image, add begin of new image
+                            mergedList.add(imageList.get(i + 1));
+                            countElementsInList++;
+                            beginImage = false;
+                        }
+                    }
+                }
+            }
+            return mergedList;
+        } else {
+            return imageList;
+        }
+    }
+
+    private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2) {
+
+        // diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten
+        double width = image1.getPosition().getWidth();
+        double height1 = image1.getPosition().getHeight();
+        double height2 = image2.getPosition().getHeight();
+        // mit  den Werten, die unter Image gespeichert sind, funktioniert es
+        double img1height = image1.getImage().getHeight();
+        double img1width = image1.getImage().getWidth();
+        double img2height = image2.getImage().getHeight();
+
+        BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB);
+        Graphics mergedImageGraphics = mergedImage.getGraphics();
+        try {
+            mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null);
+            mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null);
+
+            // set Image, Position and type for merged Image
+            //set position for merged image with values of image1 and the height of both
+            Rectangle2D pos = new Rectangle2D.Float();
+            pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), width, height1 + height2);
+            PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage());
+            // Graphics need to be disposed
+
+            image1.getImage().flush();
+            image2.getImage().flush();
+
+            mergedImage.flush();
+            mergedImageGraphics.dispose();
+
+            return newPdfImage;
+        } catch (Exception e) {
+            // failed to merge image
+            log.error("Failed to merge image", e);
+            return null;
+        }
+
+
+    }
+
+    //make a list of true and false, if the image is a candidate for merging
+    private List<Boolean> getCandidatesList(List<PdfImage> imageList) {
+        List<Boolean> candidatesList = new ArrayList<>();
+        for (int i = 0; i < imageList.size(); i++) {
+            if (i >= 1) {
+                candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i)));
+            }
+        }
+        return candidatesList;
+    }
+
+    // evaluate if two images are candidates for merging, depending on their coordinates, width and height
+    private boolean isCandidateForMerging(PdfImage image1, PdfImage image2) {
+        double x1 = image1.getPosition().getX();
+        double y1 = image1.getPosition().getY();
+        double width1 = image1.getPosition().getWidth();
+        double x2 = image2.getPosition().getX();
+        double y2 = image2.getPosition().getY();
+        double width2 = image2.getPosition().getWidth();
+        double height2 = image2.getPosition().getHeight();
+        //if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates,
+        // then it is the same picture and has to be merged -> return true
+        return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(y1 - y2) && width2 > (height2 / 6);
+    }
+

    private void increaseDocumentStatistics(Page page, Document document) {

@@ -99,11 +99,5 @@ public class RedactionStorageService {
            return projectId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
        }

-
-        public static String getStorageId(String userId, String projectId, String filename) {
-
-            return userId + "/" + projectId + "/" + filename;
-        }
-
    }
 }
@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;

 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
+
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -18,6 +20,8 @@ public abstract class AbstractTextContainer {
    protected String classification;
    protected int page;

+    private Orientation orientation = Orientation.NONE;
+
    public abstract String getText();

    public boolean contains(AbstractTextContainer other) {
@@ -246,8 +246,12 @@ public class Ruling extends Line2D.Float {

    public Ruling expand(float amount) {
        Ruling r = (Ruling) this.clone();
-        r.setStart(this.getStart() - amount);
-        r.setEnd(this.getEnd() + amount);
+        try {
+            r.setStart(this.getStart() - amount);
+            r.setEnd(this.getEnd() + amount);
+        } catch (UnsupportedOperationException e){
+            log.warn("Could not expand ruling!");
+        }
        return r;
    }

@@ -102,7 +102,7 @@ public class PdfVisualisationService {

            contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());

-            contentStream.showText(textBlock.getClassification());
+            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());

            contentStream.endText();
        }
@@ -2,7 +2,7 @@ server:
  port: 8083

 configuration-service.url: "http://localhost:8081"
-
+file-management-service.url: "http://localhost:8085"

 storage:
  bucket-name: 'redaction'
@@ -2,6 +2,7 @@ info:
  description: Redaction Service Server V1

 configuration-service.url: "http://configuration-service-v1:8080"
+file-management-service.url: "http://file-management-service-v1:8080"
 image-service.url: "http://image-service-v1:8080"

 server:
@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.configuration.v1.api.model.*;
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
 import com.iqser.red.service.file.management.v1.api.model.FileType;
 import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -11,6 +12,7 @@ import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClien
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -19,6 +21,7 @@ import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -28,10 +31,13 @@ import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
@@ -45,10 +51,10 @@ import java.util.stream.Collectors;

 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
-import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;

@RunWith(SpringRunner.class)
-@SpringBootTest(webEnvironment = RANDOM_PORT)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationTest {

    private static final String RULES = loadFromClassPath("drools/rules.drl");
@@ -68,6 +74,7 @@ public class RedactionIntegrationTest {
    private static final String SIGNATURE = "signature";
    private static final String FORMULA = "formula";
    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";

    private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
    private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -76,9 +83,13 @@ public class RedactionIntegrationTest {

    private static final String PII = "PII";

+
    @Autowired
    private RedactionController redactionController;

+    @Autowired
+    private ReanalyzeService reanalyzeService;
+
    @Autowired
    private ObjectMapper objectMapper;

@@ -104,6 +115,7 @@ public class RedactionIntegrationTest {
    private RabbitTemplate rabbitTemplate;

    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
    private final Map<String, String> typeColorMap = new HashMap<>();
    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
@@ -116,7 +128,8 @@ public class RedactionIntegrationTest {
    private final static String TEST_PROJECT_ID = "123";
    private final static String TEST_FILE_ID = "123";

-    @TestConfiguration
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
    public static class RedactionIntegrationTestConfiguration {

        @Bean
@@ -141,6 +154,7 @@ public class RedactionIntegrationTest {
            return new FileSystemBackedStorageService();
        }

+
    }


@@ -160,30 +174,45 @@ public class RedactionIntegrationTest {

        loadDictionaryForTest();
        loadTypeForTest();
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(TypeResponse.builder()
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L);
+        when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder()
                .types(getTypeResponse())
                .build());
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
-        when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(ADDRESS));
-        when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(AUTHOR));
-        when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SPONSOR));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(HINT_ONLY));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(MUST_REDACT));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(TEST_METHOD));
-        when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PII));
-        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
-        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
-        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
-        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
-        when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
-        when(dictionaryClient.getDictionaryForType(OCR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(OCR));
-        when(dictionaryClient.getDictionaryForType(LOGO, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(LOGO));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SIGNATURE));
-        when(dictionaryClient.getDictionaryForType(FORMULA, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FORMULA));
+
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypes(TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(TypeResponse.builder()
+                .types(List.of(TypeResult.builder()
+                        .type(DOSSIER_REDACTIONS)
+                        .ruleSetId(TEST_RULESET_ID)
+                        .hexColor( "#ffe187")
+                        .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                        .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                        .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                        .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                        .build()))
+                .build());
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_RULESET_ID, TEST_PROJECT_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
        when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
    }

@@ -290,6 +319,11 @@ public class RedactionIntegrationTest {
                        .stream()
                        .map(this::cleanDictionaryEntry)
                        .collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
    }


@@ -342,6 +376,7 @@ public class RedactionIntegrationTest {
        hintTypeMap.put(FORMULA, false);
        hintTypeMap.put(LOGO, false);
        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);

        caseInSensitiveMap.put(VERTEBRATE, true);
        caseInSensitiveMap.put(ADDRESS, false);
@@ -363,6 +398,7 @@ public class RedactionIntegrationTest {
        caseInSensitiveMap.put(SIGNATURE, true);
        caseInSensitiveMap.put(LOGO, true);
        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);

        recommendationTypeMap.put(VERTEBRATE, false);
        recommendationTypeMap.put(ADDRESS, false);
@@ -384,6 +420,7 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(FORMULA, false);
        recommendationTypeMap.put(SIGNATURE, false);
        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);

        rankTypeMap.put(FALSE_POSITIVE, 160);
        rankTypeMap.put(PURITY, 155);
@@ -405,6 +442,7 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(LOGO, 28);
        rankTypeMap.put(SIGNATURE, 27);
        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);

        colors.setDefaultColor("#acfc00");
        colors.setNotRedacted("#cccccc");
@@ -431,11 +469,11 @@ public class RedactionIntegrationTest {
    }


-    private DictionaryResponse getDictionaryResponse(String type) {
+    private DictionaryResponse getDictionaryResponse(String type, boolean isDossierDictionary) {

        return DictionaryResponse.builder()
                .hexColor(typeColorMap.get(type))
-                .entries(toDictionaryEntry(dictionary.get(type)))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
                .isHint(hintTypeMap.get(type))
                .isCaseInsensitive(caseInSensitiveMap.get(type))
                .isRecommendation(recommendationTypeMap.get(type))
@@ -458,21 +496,68 @@ public class RedactionIntegrationTest {
    public void test270Rotated() {
        AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf");
        MemoryStats.printMemoryStats();
-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);
        assertThat(result).isNotNull();
    }


    @Test
+    @Ignore
    public void testLargeScannedFileOOM() {
        AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
        MemoryStats.printMemoryStats();
-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);
        assertThat(result).isNotNull();
    }

+    @Test
+    public void testMergedImages() throws IOException {
+
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/merge_images.pdf");
+
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+        AnalyzeResult result = reanalyzeService.analyze(request);
+
+        Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID);
+
+        redactionLog.getRedactionLogEntry().forEach(entry -> {
+            duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
+        });
+
+        duplicates.entrySet().forEach(entry -> {
+            assertThat(entry.getValue().size()).isEqualTo(1);
+        });
+
+        dictionary.get(AUTHOR).add("Drinking water");
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L);
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .projectId(TEST_PROJECT_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated3.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long rstart = System.currentTimeMillis();
+        reanalyzeService.reanalyze(request);
+
+        long rend = System.currentTimeMillis();
+        System.out.println("reanalysis analysis duration: " + (rend - rstart));
+
+
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+
+
+    }

    @Test
+    @Ignore
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

        long start = System.currentTimeMillis();
@@ -488,7 +573,7 @@ public class RedactionIntegrationTest {

            AnalyzeRequest request = prepareStorage(new FileInputStream((path)));
            System.out.println("Redacting file : " + path.getName());
-            AnalyzeResult result = redactionController.analyze(request);
+            AnalyzeResult result = reanalyzeService.analyze(request);

            Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();

@@ -503,10 +588,10 @@ public class RedactionIntegrationTest {
            });

            dictionary.get(AUTHOR).add("Drinking water");
-            when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L);
+            when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(1L);

            long rstart = System.currentTimeMillis();
-            redactionController.reanalyze(request);
+            reanalyzeService.reanalyze(request);

            long rend = System.currentTimeMillis();
            System.out.println("reanalysis analysis duration: " + (rend - rstart));
@@ -545,7 +630,7 @@ public class RedactionIntegrationTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");
        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());

-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);

        var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID);
        var text = redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID);
@@ -591,15 +676,15 @@ public class RedactionIntegrationTest {
        dictionary.get(AUTHOR).add("physical");
        reanlysisVersions.put("physical", 2L);

-        dictionary.get(VERTEBRATE).add("s-metolachlor");
-        reanlysisVersions.put("s-metolachlor", 3L);
+//        dictionary.get(VERTEBRATE).add("s-metolachlor");
+//        reanlysisVersions.put("s-metolachlor", 3L);

-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(3L);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));

        start = System.currentTimeMillis();
-        AnalyzeResult reanalyzeResult = redactionController.reanalyze(request);
+        AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request);

        end = System.currentTimeMillis();
        System.out.println("reanalysis analysis duration: " + (end - start));
@@ -623,7 +708,7 @@ public class RedactionIntegrationTest {
        long start = System.currentTimeMillis();

        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);

        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
                .projectId(TEST_PROJECT_ID)
@@ -685,7 +770,7 @@ public class RedactionIntegrationTest {

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
        request.setManualRedactions(manualRedactions);
-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);

        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
@@ -693,7 +778,7 @@ public class RedactionIntegrationTest {
                .status(Status.APPROVED)
                .build()));

-        redactionController.reanalyze(request);
+        reanalyzeService.reanalyze(request);


        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
@@ -716,7 +801,7 @@ public class RedactionIntegrationTest {
    public void classificationTest() throws IOException {

        System.out.println("classificationTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");


        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
@@ -808,7 +893,7 @@ public class RedactionIntegrationTest {

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());

-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);

        var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID);

@@ -855,7 +940,7 @@ public class RedactionIntegrationTest {

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());

-        AnalyzeResult result = redactionController.analyze(request);
+        AnalyzeResult result = reanalyzeService.analyze(request);

        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
                .projectId(TEST_PROJECT_ID)
@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;

 import com.amazonaws.services.s3.AmazonS3;
 import com.iqser.red.service.configuration.v1.api.model.*;
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
+import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -21,10 +23,13 @@ import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
@@ -42,8 +47,9 @@ import java.util.concurrent.atomic.AtomicLong;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;

-@SpringBootTest
@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(EntityRedactionServiceTest.RedactionIntegrationTestConfiguration.class)
 public class EntityRedactionServiceTest {

    private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
@@ -74,7 +80,8 @@ public class EntityRedactionServiceTest {

    private final static String TEST_RULESET_ID = "123";

-    @TestConfiguration
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
    public static class RedactionIntegrationTestConfiguration {

        @Bean
@@ -107,8 +114,8 @@ public class EntityRedactionServiceTest {
    public void testNestedEntitiesRemoval() {

        Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false);
        entities.add(nested);
        entities.add(nesting);
        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -127,20 +134,20 @@ public class EntityRedactionServiceTest {
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                .build();
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);

        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);

        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
    }
@@ -154,19 +161,19 @@ public class EntityRedactionServiceTest {
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H.")))
                .build();
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA")))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);

        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
    }
@@ -177,21 +184,21 @@ public class EntityRedactionServiceTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
                " Supplement - Identity of the active substance - Reference list.pdf");
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()
                .entrySet()
                .stream()
@@ -199,7 +206,7 @@ public class EntityRedactionServiceTest {
        pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " +
                "the plant protection product.pdf");
        classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()
                .entrySet()
                .stream()
@@ -210,21 +217,21 @@ public class EntityRedactionServiceTest {
    public void testFalsePositiveInWrongCell() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1).stream()
                .filter(entity -> entity.getMatchedRule() == 9)
@@ -277,21 +284,21 @@ public class EntityRedactionServiceTest {
        droolsExecutionService.updateRules(TEST_RULESET_ID);

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1).stream()
                .filter(entity -> entity.getMatchedRule() == 6)
@@ -316,21 +323,21 @@ public class EntityRedactionServiceTest {
        droolsExecutionService.updateRules(TEST_RULESET_ID);

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse authorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(authorResponse);
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt"))))
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1).stream()
                .filter(entity -> entity.getMatchedRule() == 11)
@@ -348,19 +355,19 @@ public class EntityRedactionServiceTest {
                .entries(toDictionaryEntry(Arrays.asList("Bissig R.", "Thanei P.")))
                .build();

-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);

        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
        assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
        assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y
@@ -371,15 +378,15 @@ public class EntityRedactionServiceTest {
                .entries(toDictionaryEntry(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C.")))
                .build();

-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);

        classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
        assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
@@ -396,19 +403,19 @@ public class EntityRedactionServiceTest {
                .entries(toDictionaryEntry(Collections.singletonList("Aldershof S.")))
                .build();

-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .entries(toDictionaryEntry(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland")))
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);

        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .entries(Collections.emptyList())
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);
        Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
-        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
+        entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null, "dossierId");
        assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
        assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6);
    }
@@ -447,19 +454,19 @@ public class EntityRedactionServiceTest {
                        TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(),
                        TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build()))
                .build();
-        when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(typeResponse);
+        when(dictionaryClient.getVersion(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getAllTypes(TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(typeResponse);

        // Default empty return to prevent NPEs
        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                .build();
-        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse);
        DictionaryResponse addressResponse = DictionaryResponse.builder()
                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse);
        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
                .build();
-        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse);

        Colors colors = new Colors();
        colors.setDefaultColor("#acfc00");
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

 import com.amazonaws.services.s3.AmazonS3;
+import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
@@ -13,9 +14,14 @@ import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;

@@ -29,8 +35,10 @@ import java.util.stream.Collectors;

 import static org.assertj.core.api.Assertions.assertThat;

-@SpringBootTest
+
@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(PdfSegmentationServiceTest.TestConfiguration.class)
 public class PdfSegmentationServiceTest {

    @Autowired
@@ -51,6 +59,26 @@ public class PdfSegmentationServiceTest {
    @MockBean
    private AmazonS3 amazonS3;

+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = { RabbitAutoConfiguration.class})
+    public static class TestConfiguration {
+
+    }
+
+    @Test
+    public void testMergeImages() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/270Rotated.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
+        assertThat(document.getPages().get(0).getImages().size()).isEqualTo(1);
+        assertThat(document.getPages().get(1).getImages().size()).isEqualTo(0);
+
+    }
+
    @Test
    @Ignore
    public void testExtractImages() throws IOException {
@@ -1,5 +1,6 @@
 configuration-service.url: "http://configuration-service-v1:8080"
 image-service.url: "http://image-service-v1:8080"
+file-management-service.url: "http://file-management-service-v1:8080"

 ribbon:
  ConnectTimeout: 600000
@@ -1652,3 +1652,5 @@ Zoecon Corp.
 Zoecon Corp., Palo Alto, USA
 Zyma SA
 Zyma SA, Nyon, Switzerland
+Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
+Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
@@ -0,0 +1 @@
+Difenoconazole
@@ -328,4 +328,12 @@ rule "28: Redact Logos"
        Section(matchesImageType("logo"))
    then
        section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
+    end
+
+
+rule "29: Redact Dossier Redactions"
+     when
+         Section(matchesType("dossier_redactions"))
+     then
+         section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+     end
Author	SHA1	Message	Date
Dominique Eifländer	a14f2fa4cf	Fixed endless processing	2021-06-02 13:49:48 +02:00
Dominique Eifländer	69f68273d1	Ignore failing Ruling expansion	2021-06-02 13:00:51 +02:00
Dominique Eifländer	541a55798c	Fixed text parsing orientation problem	2021-06-02 11:19:34 +02:00
Dominique Eifländer	5a9b86ba76	RED-1535: Improved document parsing, added orientation to textblock	2021-06-01 15:41:14 +02:00
Dominique Eifländer	d18003736b	Fixed missing whitespaces	2021-06-01 15:40:41 +02:00
Dominique Eiflaender	b16b56daf1	Pull request #154 : RED-1477: Fixed duplicate images Merge in RED/redaction-service from RED-1477 to master * commit 'bdf40f5e4eec9984306516790eb8c4eb106b0aa1': RED-1477: Fixed duplicate images	2021-05-12 12:23:03 +02:00
Dominique Eifländer	bdf40f5e4e	RED-1477: Fixed duplicate images	2021-05-12 11:28:17 +02:00
Dominique Eiflaender	e281ab98fa	Pull request #153 : Fixed dossier dictionary renalaysis Merge in RED/redaction-service from dossierReanalysisFix to master * commit '184f55535aa07123a80a78e7a0aff949c2827465': Fixed dossier dictionary renalaysis	2021-04-30 14:00:54 +02:00
Dominique Eifländer	184f55535a	Fixed dossier dictionary renalaysis	2021-04-30 13:52:46 +02:00
Dominique Eiflaender	979ab71090	Pull request #152 : Added isDossierDictionaryEntry to manualRedactions and redactionLog Merge in RED/redaction-service from isDossierDictionary to master * commit 'b7ae6cf390b1aa58c9f6210e3c8fcbfac89a6610': Added isDossierDictionaryEntry to manualRedactions and redactionLog	2021-04-29 12:36:09 +02:00
Dominique Eifländer	b7ae6cf390	Added isDossierDictionaryEntry to manualRedactions and redactionLog	2021-04-29 12:29:50 +02:00
Dominique Eiflaender	01eceaaef4	Pull request #151 : RED-1334: Enabled to use dictionaries per dossier Merge in RED/redaction-service from RED-1334 to master * commit '48beb984ca02ab163a64af38907a05d70b048655': RED-1334: Enabled to use dictionaries per dossier	2021-04-28 15:20:50 +02:00
Dominique Eifländer	48beb984ca	RED-1334: Enabled to use dictionaries per dossier	2021-04-28 15:11:11 +02:00
Timo Bejan	f7af37d287	Pull request #150 : cleaned up image code Merge in RED/redaction-service from image-cleanup to master * commit '1ddcafbdd23adb8354259660e25723449fb47855': cleaned up image code	2021-04-26 18:35:14 +02:00
Timo	1ddcafbdd2	cleaned up image code	2021-04-26 19:01:01 +03:00
Timo Bejan	e1b8ca4956	Pull request #149 : moved changelog to redaction service, removed filename from changelog Merge in RED/redaction-service from improved-redaction-performance to master * commit 'c05fdb48545b829ae82502c6d6ad486f2210f8f2': moved changelog to redaction service, removed filename from changelog	2021-04-26 14:52:44 +02:00
Timo	c05fdb4854	moved changelog to redaction service, removed filename from changelog	2021-04-26 15:48:17 +03:00
Clarissa Dietrich	e723208473	Pull request #148 : RED-1378: fix: merging images condition Merge in RED/redaction-service from mergingFix to master * commit '047ac9bf848d168b2a1bbe9217538b6bdac00de6': RED-1378: fix: merging images condition	2021-04-26 12:16:10 +02:00
Clarissa.Dietrich	047ac9bf84	RED-1378: fix: merging images condition	2021-04-26 10:50:35 +02:00
Timo Bejan	b83250f161	Pull request #141 : fix: merge images into one Merge in RED/redaction-service from classimg3 to master * commit '4d1ad3b5a56915e9e5df9191a535c0e5cd83972c': RED-1351: merge images into one	2021-04-23 09:36:04 +02:00
Clarissa.Dietrich	4d1ad3b5a5	RED-1351: merge images into one	2021-04-22 17:10:12 +02:00
Timo Bejan	24f33fe6bf	Pull request #147 : fixed changelog storage Merge in RED/redaction-service from improved-redaction-performance to master * commit '44ff9cb7757034a3df9c017f58a3381b52d73133': fixed changelog storage	2021-04-21 19:45:17 +02:00
Timo	44ff9cb775	fixed changelog storage	2021-04-21 20:41:18 +03:00
Timo Bejan	88524d0118	Pull request #146 : reworked api Merge in RED/redaction-service from redaction-system-queue-prep to master * commit '735cfdd6c4f8ffd6024f9803c7c5466205ebf7b4': reworked api	2021-04-21 17:57:05 +02:00
Timo	735cfdd6c4	reworked api	2021-04-21 18:53:29 +03:00
Timo Bejan	08f3ee8544	Pull request #145 : removed long test, fixed NPE Merge in RED/redaction-service from redaction-system-queue-prep to master * commit '3af121193214d250fe9dc58056f13a315d724184': fixed reanalyse condition fixed reanalyse condition removed long test, fixed NPE	2021-04-21 17:29:41 +02:00
Timo	3af1211932	fixed reanalyse condition	2021-04-21 18:16:39 +03:00
Timo	c1d5be8f63	fixed reanalyse condition	2021-04-21 18:13:35 +03:00
Timo	567bb00377	removed long test, fixed NPE	2021-04-21 18:13:35 +03:00
Timo Bejan	d7679c8c38	Pull request #144 : Updated code to respond to file-status service on successful/failed analysis Merge in RED/redaction-service from redaction-system-queue-prep to master * commit 'f6d0b2d0db867afcb8a3edfd48b1d101c681cd65': temporary ignore this added missing config yml Updated code to respond to file-status service on successful/failed analysis	2021-04-21 17:01:09 +02:00
Timo	f6d0b2d0db	temporary ignore this	2021-04-21 17:34:01 +03:00
Timo	43217e6abe	added missing config yml	2021-04-21 17:28:01 +03:00
Timo	8182b1a0fa	Updated code to respond to file-status service on successful/failed analysis	2021-04-21 17:15:49 +03:00
Timo Bejan	b22e3d93c0	Pull request #143 : prepared redaction service for queue handling Merge in RED/redaction-service from redaction-system-queue-prep to master * commit 'ab3b30b28256a93fff79b642080ddc1d940992a9': prepared redaction service for queue handling	2021-04-21 13:55:48 +02:00