Made dictionaries Theadsafe

2020-12-09 16:23:27 +01:00
24 changed files with 296 additions and 689 deletions
@@ -32,9 +32,5 @@ public class RedactionLogEntry {
    private boolean manual;
    private Status status;
    private ManualRedactionType manualRedactionType;
-    private boolean isDictionaryEntry;
-
-    private String textBefore;
-    private String textAfter;

 }
@@ -5,17 +5,15 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

+import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
 public class ClassificationService {
@@ -30,7 +28,7 @@ public class ClassificationService {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

-        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
+        System.out.println(document.getFontSizeCounter().getCountPerValue());

        for (Page page : document.getPages()) {
            Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
@@ -41,7 +39,6 @@ public class ClassificationService {


    public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
-
        for (AbstractTextContainer textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextBlock) {
                classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
@@ -50,33 +47,24 @@ public class ClassificationService {
    }


-    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
-                              List<Float> headlineFontSizes) {
-
+    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
        if (document.getFontSizeCounter().getMostPopular() == null) {
            // TODO Figure out why this happens.
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification("Header");

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
-                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
-                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
-                .getMostPopular() || page.getTextBlocks().size() == 1)) {
+        }  else if (page.getPageNumber() == 1
+                && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
+                && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                textBlock.setClassification("Title");
            }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
-                .getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
-                .equals("bold")) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {

            for (int i = 1; i <= headlineFontSizes.size(); i++) {
                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
@@ -84,34 +72,20 @@ public class ClassificationService {
                    document.setHeadlines(true);
                }
            }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
-                .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        }else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
            textBlock.setClassification("TextBlock Bold");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter()
-                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular()) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
            textBlock.setClassification("TextBlock");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("TextBlock Italic");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
            textBlock.setClassification("TextBlock Unknown");
        }
    }
-
 }
@@ -24,12 +24,11 @@ import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationSer
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@RestController
@RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
@@ -38,6 +37,7 @@ public class RedactionController implements RedactionResource {
    private final PdfSegmentationService pdfSegmentationService;
    private final AnnotationHighlightService annotationHighlightService;
    private final EntityRedactionService entityRedactionService;
+    private final PdfFlattenService pdfFlattenService;
    private final DroolsExecutionService droolsExecutionService;


@@ -48,14 +48,15 @@ public class RedactionController implements RedactionResource {
            pdDocument.setAllSecurityToBeRemoved(true);

            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-
-            log.info("Document structure analysis successful, starting redaction analysis...");
-
            entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
            annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
                    .getManualRedactions());

-            log.info("Redaction analysis successful...");
+            if (redactionRequest.isFlatRedaction()) {
+                PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
+                return convert(flatDocument, classifiedDoc.getPages()
+                        .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
+            }

            return convert(pdDocument, classifiedDoc.getPages()
                    .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
@@ -212,7 +212,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                maxCharWidths = charWidth;
            }

-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
+            if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
                startIndex++;
                continue;
            }
@@ -220,15 +220,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i;
            }

-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
+            if (i > 0 && textPositions.get(i).getUnicode().equals(" ") && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i + 1;
@@ -236,13 +236,14 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        }

        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
+        if (!sublist.isEmpty() && sublist.get(sublist.size() - 1).getUnicode().equals(" ")) {
            sublist = sublist.subList(0, sublist.size() - 1);
        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+        if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
        }
        super.writeString(text);
+
    }

    @Override
@@ -1,8 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.Iterator;
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -12,9 +9,9 @@ import lombok.Value;
@Value
 public class CellValue {

-    private List<TextBlock> textBlocks;
+    TextBlock textBlock;

-    private int rowSpanStart;
+    int rowSpanStart;


    @Override
@@ -22,24 +19,17 @@ public class CellValue {

        StringBuilder sb = new StringBuilder();

-        Iterator<TextBlock> itty = textBlocks.iterator();
-        while (itty.hasNext()) {
-            TextBlock textBlock = itty.next();
-            TextPositionSequence previous = null;
-            for (TextPositionSequence word : textBlock.getSequences()) {
-                if (previous != null) {
-                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
-                        sb.append('\n');
-                    } else {
-                        sb.append(' ');
-                    }
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : textBlock.getSequences()) {
+            if (previous != null) {
+                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
                }
-                sb.append(word.toString());
-                previous = word;
-            }
-            if (itty.hasNext()) {
-                sb.append(' ');
            }
+            sb.append(word.toString());
+            previous = word;
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
@@ -1,33 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.Getter;

@Data
+@AllArgsConstructor
 public class Dictionary {

    public static final String RECOMMENDATION_PREFIX = "recommendation_";

    @Getter
    private List<DictionaryModel> dictionaryModels;
-    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
+    private Map<String, DictionaryModel> localAccessMap;

    @Getter
    private long version;


-    public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
-        this.dictionaryModels = dictionaryModels;
-        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
-        this.version = dictionaryVersion;
-    }
-
-
    public boolean isRecommendation(String type) {

        DictionaryModel model = localAccessMap.get(type);
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;


-import java.io.Serializable;
 import java.util.Set;

 import lombok.AllArgsConstructor;
@@ -9,7 +8,7 @@ import lombok.Data;

@Data
@AllArgsConstructor
-public class DictionaryModel implements Serializable {
+public class DictionaryModel {

    private String type;
    private int rank;
@@ -12,7 +12,7 @@ import lombok.EqualsAndHashCode;
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {

-
+    @EqualsAndHashCode.Include
    private final String word;
    private final String type;
    private boolean redaction;
@@ -20,10 +20,7 @@ public class Entity {
    private String legalBasis;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
    private List<TextPositionSequence> targetSequences;
-
-    @EqualsAndHashCode.Include
    private Integer start;
-    @EqualsAndHashCode.Include
    private Integer end;

    @EqualsAndHashCode.Include
@@ -33,13 +30,8 @@ public class Entity {
    @EqualsAndHashCode.Include
    private int sectionNumber;

-    private boolean isDictionaryEntry;

-    private String textBefore;
-    private String textAfter;
-
-
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {

        this.word = word;
        this.type = type;
@@ -50,13 +42,10 @@ public class Entity {
        this.matchedRule = matchedRule;
        this.sectionNumber = sectionNumber;
        this.legalBasis = legalBasis;
-        this.isDictionaryEntry = isDictionaryEntry;
-        this.textBefore = textBefore;
-        this.textAfter = textAfter;
    }


-    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {

        this.word = word;
        this.type = type;
@@ -64,7 +53,6 @@ public class Entity {
        this.end = end;
        this.headline = headline;
        this.sectionNumber = sectionNumber;
-        this.isDictionaryEntry = isDictionaryEntry;
    }

 }
@@ -2,20 +2,18 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;

 import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;

-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;

 import org.apache.commons.lang3.StringUtils;

-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil;

 import lombok.Builder;
 import lombok.Data;
@@ -47,17 +45,14 @@ public class Section {

    private Map<String, CellValue> tabularData;

-    private Dictionary dictionary;
-
-    private SearchableText searchableText;
-

    public boolean rowEquals(String headerName, String value) {

        String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");

        return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
-                .toString()
+                .getTextBlock()
+                .getText()
                .equals(value);
    }

@@ -194,13 +189,7 @@ public class Section {
            for (String value : values) {
                if (StringUtils.isNotBlank(value)) {
                    String[] lines = value.split("\n");
-
                    for (String line : lines) {
-
-                        if (line.trim().length() <= 2) {
-                            return;
-                        }
-
                        Set<Entity> found = findEntities(line.trim(), asType, false);

                        // HashSet keeps the older value, but we want the new only.
@@ -228,10 +217,6 @@ public class Section {

    private Set<Entity> findEntities(String value, String asType, boolean caseinsensitive) {

-        if (value.trim().length() <= 2) {
-            return new HashSet<>();
-        }
-
        Set<Entity> found = new HashSet<>();

        String text = caseinsensitive ? searchText.toLowerCase() : searchText;
@@ -245,11 +230,11 @@ public class Section {

            if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(text.charAt(startIndex - 1)) || isSeparator(text
                    .charAt(startIndex - 1))) && (stopIndex == text.length() || isSeparator(text.charAt(stopIndex)))) {
-                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber, false));
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
            }
        } while (startIndex > -1);

-        return PositionUtil.clearAndFindPositions(found, searchableText, dictionary);
+        return removeEntitiesContainedInLarger(found);
    }


@@ -259,6 +244,22 @@ public class Section {
    }


+    public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
+
+        List<Entity> wordsToRemove = new ArrayList<>();
+        for (Entity word : entities) {
+            for (Entity inner : entities) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                    wordsToRemove.add(inner);
+                }
+            }
+        }
+        entities.removeAll(wordsToRemove);
+        return entities;
+    }
+
+
    public void highlightCell(String cellHeader, int ruleNumber, String type) {

        annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
@@ -290,26 +291,19 @@ public class Section {
        } else {
            String word = value.toString();

-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
+            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber);
            entity.setRedaction(redact);
            entity.setMatchedRule(ruleNumber);
            entity.setRedactionReason(reason);
-            entity.setTargetSequences(value.getTextBlocks()
-                    .stream()
-                    .map(TextBlock::getSequences)
-                    .flatMap(Collection::stream)
-                    .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
+            entity.setTargetSequences(value.getTextBlock()
+                    .getSequences()); // Make sure no other cells with same content are highlighted
            entity.setLegalBasis(legalBasis);

-            Set<Entity> singleEntitySet = new HashSet<>();
-            singleEntitySet.add(entity);
-            PositionUtil.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
-
            // HashSet keeps the older value, but we want the new only.
            entities.remove(entity);
            entities.add(entity);

-            PositionUtil.removeEntitiesContainedInLarger(entities);
+            entities = removeEntitiesContainedInLarger(entities);

            if (addAsRecommendations && !isLocal()) {
                String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
@@ -11,7 +11,6 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.configuration.v1.api.model.Colors;
@@ -169,14 +168,8 @@ public class DictionaryService {
    }


-    public Dictionary getDeepCopyDictionary(){
-        List<DictionaryModel> copy = new ArrayList<>();
-
-        dictionary.forEach(dm -> {
-            copy.add(SerializationUtils.clone(dm));
-        });
-
-        return new Dictionary(copy, dictionaryVersion);
+    public Dictionary getThreadSafeDictionary(){
+        return new Dictionary(new ArrayList<>(dictionary), new HashMap<>(localAccessMap), dictionaryVersion);
    }

 }
@@ -6,7 +6,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -29,21 +28,17 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
 public class EntityRedactionService {

    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;


    public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
@@ -52,7 +47,7 @@ public class EntityRedactionService {
        droolsExecutionService.updateRules();
        long rulesVersion = droolsExecutionService.getRulesVersion();

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary();
+        Dictionary dictionary = dictionaryService.getThreadSafeDictionary();

        Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null));

@@ -71,7 +66,7 @@ public class EntityRedactionService {
            documentEntities.removeAll(foundByLocal);
            documentEntities.addAll(foundByLocal);

-            PositionUtil.removeEntitiesContainedInLarger(documentEntities);
+            removeEntitiesContainedInLarger(documentEntities);
        }

        for (Entity entity : documentEntities) {
@@ -86,7 +81,7 @@ public class EntityRedactionService {
                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
+                                .getLegalBasis()));
            }
        }

@@ -97,185 +92,125 @@ public class EntityRedactionService {
    }


-    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, Dictionary dictionary,
-                                     boolean local, Map<Integer, Set<Entity>> hintsPerSectionNumber) {
+    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions,
+                                     Dictionary dictionary, boolean local,
+                                     Map<Integer, Set<Entity>> hintsPerSectionNumber) {

        Set<Entity> documentEntities = new HashSet<>();
-
-        AtomicInteger sectionNumber = new AtomicInteger(1);
+        int sectionNumber = 1;
        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {

+            SearchableText searchableText = paragraph.getSearchableText();
+
            List<Table> tables = paragraph.getTables();
+
            for (Table table : tables) {
-                if (table.getColCount() == 2) {
-                    sectionSearchableTextPairs.addAll(processTableAsOneText(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
-                } else {
-                    sectionSearchableTextPairs.addAll(processTablePerRow(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+                boolean singleCellTable = table.getRowCount() == 1 && table.getColCount() == 1;
+                for (List<Cell> row : table.getRows()) {
+                    SearchableText searchableRow = new SearchableText();
+                    Map<String, CellValue> tabularData = new HashMap<>();
+                    int start = 0;
+                    for (Cell cell : row) {
+                        if (!singleCellTable && cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
+                            continue;
+                        }
+                        addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber);
+                        int cellStart = start;
+                        cell.getHeaderCells().forEach(headerCell -> {
+                            StringBuilder headerBuilder = new StringBuilder();
+                            headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
+                            String headerName = headerBuilder.toString()
+                                    .replaceAll("\n", "")
+                                    .replaceAll(" ", "")
+                                    .replaceAll("-", "");
+                            tabularData.put(headerName, new CellValue(cell.getTextBlocks().get(0), cellStart));
+                        });
+                        start = start + cell.toString().length() + 1; // include automatically appended white space
+                        for (TextBlock textBlock : cell.getTextBlocks()) {
+                            searchableRow.addAll(textBlock.getSequences());
+                        }
+                    }
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
+
+                    sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                            .isLocal(local)
+                            .dictionaryTypes(dictionary.getTypes())
+                            .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
+                                    .concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
+                                    .collect(Collectors.toSet()) : rowEntities)
+                            .text(searchableRow.getAsStringWithLinebreaks())
+                            .searchText(searchableRow.toString())
+                            .headline(table.getHeadline())
+                            .sectionNumber(sectionNumber)
+                            .tabularData(tabularData)
+                            .build(), searchableRow));
+
+                    sectionNumber++;
                }
-                sectionNumber.incrementAndGet();
+                sectionNumber++;
            }
-            sectionSearchableTextPairs.add(processText(paragraph, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
-            sectionNumber.incrementAndGet();
+
+            addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
+            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
+
+            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                    .isLocal(local)
+                    .dictionaryTypes(dictionary.getTypes())
+                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
+                            .concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
+                            .collect(Collectors.toSet()) : entities)
+                    .text(searchableText.getAsStringWithLinebreaks())
+                    .searchText(searchableText.toString())
+                    .headline(paragraph.getHeadline())
+                    .sectionNumber(sectionNumber)
+                    .build(), searchableText));
+
+            sectionNumber++;
        }

        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
            Section analysedRowSection = droolsExecutionService.executeRules(sectionSearchableTextPair.getSection());
-            documentEntities.addAll(analysedRowSection.getEntities());
+            documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair.getSearchableText(), dictionary));

            analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-                if (dictionary.isRecommendation(key)) {
+                if (dictionary.isRecommendation(key)){
                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-                        if (!dictionary.containsValue(key, value)) {
+                        if (!dictionary.containsValue(key, value)){
                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                        }
                    });
                } else {
-                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-
-                        if (dictionary.getLocalAccessMap().get(key) == null) {
-                            log.warn("Dictionary {} is null", key);
-                        }
-
-                        if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                            log.warn("Dictionary {} localEntries is null", key);
-                        }
-
-                        dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
+                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                    });
                }
            });

+
        });

        return documentEntities;
    }


-    private List<SectionSearchableTextPair> processTablePerRow(Table table, ManualRedactions manualRedactions,
-                                                               AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
-                                                               Map<Integer, Set<Entity>> hintsPerSectionNumber) {
+    private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {

-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-        for (List<Cell> row : table.getRows()) {
-            SearchableText searchableRow = new SearchableText();
-            Map<String, CellValue> tabularData = new HashMap<>();
-            int start = 0;
-            List<Integer> cellStarts = new ArrayList<>();
-            for (Cell cell : row) {
-                if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
-                    continue;
-                }
-                addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
-                int cellStart = start;
-                cell.getHeaderCells().forEach(headerCell -> {
-                    StringBuilder headerBuilder = new StringBuilder();
-                    headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
-                    String headerName = headerBuilder.toString()
-                            .replaceAll("\n", "")
-                            .replaceAll(" ", "")
-                            .replaceAll("-", "");
-                    tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
-                });
+        removeEntitiesContainedInLarger(entities);

-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    // TODO avoid cell overlap merging.
-                    searchableRow.addAll(textBlock.getSequences());
-                }
-                cellStarts.add(cellStart);
-                start = start + cell.toString().trim().length() + 1;
-            }
-            Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
-            surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
-
-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                    .isLocal(local)
-                    .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
-                            .concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
-                            .collect(Collectors.toSet()) : rowEntities)
-                    .text(searchableRow.getAsStringWithLinebreaks())
-                    .searchText(searchableRow.toString())
-                    .headline(table.getHeadline())
-                    .sectionNumber(sectionNumber.intValue())
-                    .tabularData(tabularData)
-                    .searchableText(searchableRow)
-                    .dictionary(dictionary)
-                    .build(), searchableRow));
-
-            sectionNumber.incrementAndGet();
-        }
-        return sectionSearchableTextPairs;
-    }
-
-
-    private List<SectionSearchableTextPair> processTableAsOneText(Table table, ManualRedactions manualRedactions,
-                                                                  AtomicInteger sectionNumber, Dictionary dictionary,
-                                                                  boolean local,
-                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber) {
-
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-        SearchableText entireTableText = new SearchableText();
-        for (List<Cell> row : table.getRows()) {
-            for (Cell cell : row) {
-                if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
-                    continue;
-                }
-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    entireTableText.addAll(textBlock.getSequences());
-                }
-                addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
+        for (Entity entity : entities) {
+            if (entity.getPositionSequences().isEmpty()) {
+                entity.setPositionSequences(text.getSequences(entity.getWord(), dictionary.isCaseInsensitiveDictionary(entity
+                        .getType()), entity.getTargetSequences()));
            }
        }

-        Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
-        surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);
-
-        sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                .isLocal(local)
-                .dictionaryTypes(dictionary.getTypes())
-                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream.concat(rowEntities
-                        .stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
-                        .collect(Collectors.toSet()) : rowEntities)
-                .text(entireTableText.getAsStringWithLinebreaks())
-                .searchText(entireTableText.toString())
-                .headline(table.getHeadline())
-                .sectionNumber(sectionNumber.intValue())
-                .searchableText(entireTableText)
-                .dictionary(dictionary)
-                .build(), entireTableText));
-
-        return sectionSearchableTextPairs;
-    }
-
-
-    private SectionSearchableTextPair processText(Paragraph paragraph, ManualRedactions manualRedactions,
-                                                  AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
-                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber) {
-
-        SearchableText searchableText = paragraph.getSearchableText();
-        addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber.intValue());
-        Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber.intValue(), dictionary, local);
-        surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
-
-        return new SectionSearchableTextPair(Section.builder()
-                .isLocal(local)
-                .dictionaryTypes(dictionary.getTypes())
-                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream.concat(entities
-                        .stream(), hintsPerSectionNumber.get(sectionNumber).stream())
-                        .collect(Collectors.toSet()) : entities)
-                .text(searchableText.getAsStringWithLinebreaks())
-                .searchText(searchableText.toString())
-                .headline(paragraph.getHeadline())
-                .sectionNumber(sectionNumber.intValue())
-                .searchableText(searchableText)
-                .dictionary(dictionary)
-                .build(), searchableText);
+        return entities;
    }


    private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
-                                     Dictionary dictionary, boolean local) {
+                                     List<DictionaryModel> dictionary, boolean local) {

        Set<Entity> found = new HashSet<>();
        String searchableString = searchableText.toString();
@@ -284,30 +219,23 @@ public class EntityRedactionService {
        }

        String lowercaseInputString = searchableString.toLowerCase();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+        for (DictionaryModel model : dictionary) {
            if (model.isCaseInsensitive()) {
-                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
            } else {
-                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
            }
        }
-
-        return PositionUtil.clearAndFindPositions(found, searchableText, dictionary);
+        removeEntitiesContainedInLarger(found);
+        return found;

    }


-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                             boolean local) {
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {

        Set<Entity> found = new HashSet<>();
-
        for (String value : values) {
-
-            if (value.trim().length() <= 2) {
-                continue;
-            }
-
            int startIndex;
            int stopIndex = 0;
            do {
@@ -316,7 +244,7 @@ public class EntityRedactionService {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
                }
            } while (startIndex > -1);
        }
@@ -330,6 +258,22 @@ public class EntityRedactionService {
    }


+    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
+        List<Entity> wordsToRemove = new ArrayList<>();
+        for (Entity word : entities) {
+            for (Entity inner : entities) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
+                        .getSectionNumber() == inner.getSectionNumber()) {
+                    wordsToRemove.add(inner);
+                }
+            }
+        }
+        entities.removeAll(wordsToRemove);
+    }
+
+
    private void addSectionToManualRedactions(List<TextBlock> textBlocks, ManualRedactions manualRedactions,
                                              String section, int sectionNumber) {

@@ -1,140 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.Set;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class SurroundingWordsService {
-
-    private final RedactionServiceSettings redactionServiceSettings;
-
-
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
-
-        if (entities.isEmpty()) {
-            return;
-        }
-
-        try {
-            for (Entity entity : entities) {
-
-                if (dictionary.isHint(entity.getType())) {
-                    continue;
-                }
-                findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
-            }
-        } catch (Exception e) {
-            log.warn("Could not get surrounding text!");
-        }
-    }
-
-
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
-                                   List<Integer> cellstarts) {
-
-        if (entities.isEmpty()) {
-            return;
-        }
-
-        try {
-            String searchableString = searchableText.toString();
-
-            if (cellstarts != null) {
-                for (int i = 0; i < cellstarts.size(); i++) {
-
-                    int startOffset = cellstarts.get(i);
-                    int endOffset = -1;
-
-                    if (i + 1 < cellstarts.size()) {
-                        endOffset = cellstarts.get(i + 1);
-                    } else {
-                        endOffset = searchableString.length() - 1;
-                    }
-
-                    String text = searchableString.substring(startOffset, endOffset);
-                    for (Entity entity : entities) {
-
-                        if (dictionary.isHint(entity.getType())) {
-                            continue;
-                        }
-
-                        if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
-                            int entityStartOffset = entity.getStart() - startOffset;
-                            int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
-                        }
-                    }
-                }
-            }
-        } catch (Exception e) {
-            log.warn("Could not get surrounding text!");
-        }
-    }
-
-
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
-
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
-                .getSurroundingWordsOffsetWindow();
-        String textBefore = text.substring(offsetBefore, entityStartOffset);
-        if (!textBefore.isBlank()) {
-            String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
-                    .getNumberOfSurroundingWords() : wordsBefore.length;
-            if (wordsBefore.length > 0) {
-                entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
-            }
-        }
-
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
-                .length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
-        String textAfter = text.substring(entityEndOffset, endOffset);
-        if (!textAfter.isBlank()) {
-            String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
-                    .getNumberOfSurroundingWords() : wordsAfter.length;
-            if (wordsAfter.length > 0) {
-                entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
-            }
-        }
-    }
-
-
-    private String concatWordsBefore(String[] words, int number, boolean endWithSpace) {
-
-        StringBuilder sb = new StringBuilder();
-
-        int startNumber = words.length > number ? words.length - number : 0;
-
-        for (int i = startNumber; i < words.length; i++) {
-            sb.append(words[i]).append(" ");
-        }
-
-        String result = sb.toString().trim();
-        return endWithSpace ? result + " " : result;
-    }
-
-
-    private String concatWordsAfter(String[] words, int number, boolean startWithSpace) {
-
-        StringBuilder sb = new StringBuilder();
-
-        for (int i = 0; i < number; i++) {
-            sb.append(words[i]).append(" ");
-        }
-
-        String result = sb.toString().trim();
-        return startWithSpace ? " " + result : result;
-    }
-
-}
@@ -1,78 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.utils;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
-import lombok.experimental.UtilityClass;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@UtilityClass
-public class PositionUtil {
-
-    public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
-
-        Map<String, List<Entity>> entitiesByWord = new HashMap<>();
-
-        for (Entity entity : entities) {
-            entitiesByWord.computeIfAbsent(entity.getWord(), (x) -> new ArrayList<>()).add(entity);
-        }
-
-        for (String word : entitiesByWord.keySet()) {
-
-            List<Entity> orderedEntities = entitiesByWord.get(word)
-                    .stream()
-                    .sorted(Comparator.comparing(Entity::getStart))
-                    .collect(Collectors.toList());
-            Entity firstEntity = orderedEntities.get(0);
-            List<EntityPositionSequence> positionSequences;
-            try {
-                positionSequences = text.getSequences(firstEntity.getWord(), dictionary.isCaseInsensitiveDictionary(firstEntity
-                        .getType()), firstEntity.getTargetSequences());
-            } catch (Exception e){
-                log.error("Problem occured at word: " + firstEntity.getWord() + "in text:" + text.toString());
-                return entities;
-            }
-
-            for (int i = 0; i <= orderedEntities.size() - 1; i++) {
-                try {
-                    orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
-                } catch (Exception e){
-                    log.warn("Mismatch between EntityPositionSequence and found Entity!");
-                }
-            }
-        }
-
-        removeEntitiesContainedInLarger(entities);
-
-        return entities;
-    }
-
-
-    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
-
-        List<Entity> wordsToRemove = new ArrayList<>();
-        for (Entity word : entities) {
-            for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord()
-                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
-                        .getSectionNumber() == inner.getSectionNumber()) {
-                    wordsToRemove.add(inner);
-                }
-            }
-        }
-        entities.removeAll(wordsToRemove);
-    }
-
-
-}
@@ -11,7 +11,7 @@ public class TextNormalizationUtilities {
     * @return Text without line-break hyphenation.
     */
    public static String removeHyphenLineBreaks(String text) {
-        return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
    }

 }
@@ -7,9 +7,12 @@ import lombok.Data;
@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
-    
-    private int numberOfSurroundingWords = 3;

-    private int surroundingWordsOffsetWindow = 100;
+    /**
+     * Tenant used in single tenant mode.
+     */
+    private String  defaultTenant = "iqser-id";
+
+    private int flattenImageDpi = 100;

 }
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;

 import java.awt.geom.Point2D;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -42,13 +41,8 @@ public class Cell extends Rectangle {

        StringBuilder sb = new StringBuilder();

-
-        Iterator<TextBlock> itty = textBlocks.iterator();
-        TextPositionSequence previous = null;
-        while (itty.hasNext()) {
-
-            TextBlock textBlock = itty.next();
-
+        for (TextBlock textBlock : textBlocks) {
+            TextPositionSequence previous = null;
            for (TextPositionSequence word : textBlock.getSequences()) {
                if (previous != null) {
                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
@@ -60,7 +54,6 @@ public class Cell extends Rectangle {
                sb.append(word.toString());
                previous = word;
            }
-
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
@@ -68,8 +61,4 @@ public class Cell extends Rectangle {
                .replaceAll(" {2}", " ");
    }

-
-
-
-
 }
@@ -1,111 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
-
-import java.util.ArrayDeque;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.List;
-
-/**
- * Copied and minimal modified from PDFBox.
- */
-public final class QuickSort {
-
-    private QuickSort() {
-
-    }
-
-
-    private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
-        @Override
-        public int compare(Comparable object1, Comparable object2) {
-
-            return object1.compareTo(object2);
-        }
-    };
-
-
-    /**
-     * Sorts the given list using the given comparator.
-     *
-     * @param <T>  type of the objects to be sorted.
-     * @param list list to be sorted
-     * @param cmp  comparator used to compare the objects within the list
-     */
-    public static <T> void sort(List<T> list, Comparator<? super T> cmp) {
-
-        int size = list.size();
-        if (size < 2) {
-            return;
-        }
-        quicksort(list, cmp);
-    }
-
-
-    /**
-     * Sorts the given list using compareTo as comparator.
-     *
-     * @param <T>  type of the objects to be sorted.
-     * @param list list to be sorted
-     */
-    public static <T extends Comparable> void sort(List<T> list) {
-
-        sort(list, (Comparator<T>) OBJCOMP);
-    }
-
-
-    private static <T> void quicksort(List<T> list, Comparator<? super T> cmp) {
-
-        Deque<Integer> stack = new ArrayDeque<Integer>();
-        stack.push(0);
-        stack.push(list.size());
-        while (!stack.isEmpty()) {
-            int right = stack.pop();
-            int left = stack.pop();
-            if (right - left < 2) {
-                continue;
-            }
-            int p = left + ((right - left) / 2);
-            p = partition(list, cmp, p, left, right);
-
-            stack.push(p + 1);
-            stack.push(right);
-
-            stack.push(left);
-            stack.push(p);
-        }
-    }
-
-
-    private static <T> int partition(List<T> list, Comparator<? super T> cmp, int p, int start, int end) {
-
-        int l = start;
-        int h = end - 2;
-        T piv = list.get(p);
-        swap(list, p, end - 1);
-
-        while (l < h) {
-            if (cmp.compare(list.get(l), piv) <= 0) {
-                l++;
-            } else if (cmp.compare(piv, list.get(h)) <= 0) {
-                h--;
-            } else {
-                swap(list, l, h);
-            }
-        }
-        int idx = h;
-        if (cmp.compare(list.get(h), piv) < 0) {
-            idx++;
-        }
-        swap(list, end - 1, idx);
-        return idx;
-    }
-
-
-    private static <T> void swap(List<T> list, int i, int j) {
-
-        T tmp = list.get(i);
-        list.set(i, list.get(j));
-        list.set(j, tmp);
-    }
-
-}
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.utils;

 import java.math.BigDecimal;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;

@@ -12,29 +13,23 @@ public class Utils {

    private final static float EPSILON = 0.1f;

-
    public static boolean feq(double f1, double f2) {
-
        return (Math.abs(f1 - f2) < EPSILON);
    }

-
    public static float round(double d, int decimalPlace) {
-
        BigDecimal bd = BigDecimal.valueOf(d);
        bd = bd.setScale(decimalPlace, BigDecimal.ROUND_HALF_UP);
        return bd.floatValue();
    }

-
-    public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
-
+	public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
        try {
-            QuickSort.sort(list, comparator);
-        } catch (IllegalArgumentException e) {
-            // This should not happen since we use QuickSort from PDFBox
+            Collections.sort(list, comparator);
+        } catch (IllegalArgumentException e){
+            //TODO Figure out why this happens.
            log.warn(e.getMessage());
        }
-    }
+	}

 }
@@ -256,7 +256,6 @@ public class AnnotationHighlightService {
                .manual(true)
                .status(manualRedactionEntry.getStatus())
                .manualRedactionType(ManualRedactionType.ADD)
-                .isDictionaryEntry(false)
                .build();
    }

@@ -275,9 +274,6 @@ public class AnnotationHighlightService {
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
-                .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
                .build();
    }

@@ -0,0 +1,68 @@
+package com.iqser.red.service.redaction.v1.server.visualization.service;
+
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
+import org.apache.pdfbox.rendering.ImageType;
+import org.apache.pdfbox.rendering.PDFRenderer;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class PdfFlattenService {
+
+    private final RedactionServiceSettings settings;
+
+    public PDDocument flattenPDF(PDDocument sourceDoc) throws IOException {
+
+        PDDocument destDoc = new PDDocument();
+
+        PDFRenderer pdfRenderer = new PDFRenderer(sourceDoc);
+
+        final int pageCount = sourceDoc.getDocumentCatalog().getPages().getCount();
+
+        log.info(pageCount + " page" + (pageCount == 1 ? "" : "s") + " to flatten.");
+
+        for (int i = 0; i < pageCount; i += 1) {
+
+            log.info("Flattening page " + (i + 1) + " of " + pageCount + "...");
+
+            BufferedImage img = pdfRenderer.renderImageWithDPI(i, settings.getFlattenImageDpi(), ImageType.RGB);
+
+            log.info("Image rendered in memory (" + img.getWidth() + "x" + img.getHeight() + " " + settings.getFlattenImageDpi() + "DPI).  Adding to PDF...");
+
+            PDPage imagePage = new PDPage(new PDRectangle(img.getWidth(), img.getHeight()));
+            destDoc.addPage(imagePage);
+
+            PDImageXObject imgObj = LosslessFactory.createFromImage(destDoc, img);
+
+            PDPageContentStream imagePageContentStream = new PDPageContentStream(destDoc, imagePage);
+            imagePageContentStream.drawImage(imgObj, 0, 0);
+
+            log.info("Image added successfully.");
+
+            imagePageContentStream.close();
+
+            img.flush();
+        }
+
+        log.info("New flattened PDF created in memory.");
+
+        sourceDoc.close();
+
+        return destDoc;
+    }
+
+}
@@ -331,7 +331,6 @@ public class RedactionIntegrationTest {
    @Test
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

-        long start = System.currentTimeMillis();
        System.out.println("noExceptionShouldBeThrownForAnyFiles");
        ClassLoader loader = getClass().getClassLoader();
        URL url = loader.getResource("files");
@@ -357,10 +356,6 @@ public class RedactionIntegrationTest {
            });
        }

-        long end = System.currentTimeMillis();
-
-        System.out.println("duration: " + (end - start));
-
    }


@@ -387,7 +382,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -396,11 +391,11 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.redact(request);

-//        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
-//            if(!entry.isHint()){
-//                System.out.println(entry.getPositions().get(0).getPage() +":"+ entry.getTextBefore() +"--->"+ entry.getValue() + "--->" + entry.getTextAfter());
-//            }
-//        });
+        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
+            if(entry.isRecommendation()){
+                System.out.println(entry.getValue());
+            }
+        });

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
            fileOutputStream.write(result.getDocument());
@@ -48,7 +48,6 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.PositionUtil;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;

@@ -105,11 +104,11 @@ public class EntityRedactionServiceTest {
    public void testNestedEntitiesRemoval() {

        Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
        entities.add(nested);
        entities.add(nesting);
-        PositionUtil.removeEntitiesContainedInLarger(entities);
+        entityRedactionService.removeEntitiesContainedInLarger(entities);

        assertThat(entities.size()).isEqualTo(1);
        assertThat(entities).contains(nesting);
@@ -405,7 +404,7 @@ public class EntityRedactionServiceTest {
            entityRedactionService.processDocument(classifiedDoc, null);
            assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
-            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
+            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(8);
        }
    }

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TextNormalizationUtilities {
+
+    /**
+     * Revert hyphenation due to line breaks.
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+    }
+
+}
@@ -10,11 +10,11 @@ public class TextNormalizationUtilitiesTest {

        String test = "Without these peo-\nple, this conference would not happen";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("people");
+                .contains("\npeople");

        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("einzelnen", "Vertrages");
+                .contains("\neinzelnen", "\nVertrages");

    }