Made dictionaries Theadsafe

2020-12-09 16:23:27 +01:00
22 changed files with 178 additions and 465 deletions
@@ -32,9 +32,5 @@ public class RedactionLogEntry {
    private boolean manual;
    private Status status;
    private ManualRedactionType manualRedactionType;
-    private boolean isDictionaryEntry;
-
-    private String textBefore;
-    private String textAfter;

 }
@@ -5,17 +5,15 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

+import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
 public class ClassificationService {
@@ -30,7 +28,7 @@ public class ClassificationService {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

-        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
+        System.out.println(document.getFontSizeCounter().getCountPerValue());

        for (Page page : document.getPages()) {
            Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
@@ -41,7 +39,6 @@ public class ClassificationService {


    public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
-
        for (AbstractTextContainer textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextBlock) {
                classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
@@ -50,33 +47,24 @@ public class ClassificationService {
    }


-    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
-                              List<Float> headlineFontSizes) {
-
+    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
        if (document.getFontSizeCounter().getMostPopular() == null) {
            // TODO Figure out why this happens.
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification("Header");

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
-                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
-                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
-                .getMostPopular() || page.getTextBlocks().size() == 1)) {
+        }  else if (page.getPageNumber() == 1
+                && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
+                && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                textBlock.setClassification("Title");
            }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
-                .getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
-                .equals("bold")) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {

            for (int i = 1; i <= headlineFontSizes.size(); i++) {
                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
@@ -84,34 +72,20 @@ public class ClassificationService {
                    document.setHeadlines(true);
                }
            }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
-                .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        }else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
            textBlock.setClassification("TextBlock Bold");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter()
-                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular()) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
            textBlock.setClassification("TextBlock");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("TextBlock Italic");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
            textBlock.setClassification("TextBlock Unknown");
        }
    }
-
 }
@@ -24,12 +24,11 @@ import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationSer
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@RestController
@RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
@@ -38,6 +37,7 @@ public class RedactionController implements RedactionResource {
    private final PdfSegmentationService pdfSegmentationService;
    private final AnnotationHighlightService annotationHighlightService;
    private final EntityRedactionService entityRedactionService;
+    private final PdfFlattenService pdfFlattenService;
    private final DroolsExecutionService droolsExecutionService;


@@ -48,14 +48,15 @@ public class RedactionController implements RedactionResource {
            pdDocument.setAllSecurityToBeRemoved(true);

            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-
-            log.info("Document structure analysis successful, starting redaction analysis...");
-
            entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
            annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
                    .getManualRedactions());

-            log.info("Redaction analysis successful...");
+            if (redactionRequest.isFlatRedaction()) {
+                PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
+                return convert(flatDocument, classifiedDoc.getPages()
+                        .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
+            }

            return convert(pdDocument, classifiedDoc.getPages()
                    .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
@@ -1,8 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.Iterator;
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -12,9 +9,9 @@ import lombok.Value;
@Value
 public class CellValue {

-    private List<TextBlock> textBlocks;
+    TextBlock textBlock;

-    private int rowSpanStart;
+    int rowSpanStart;


    @Override
@@ -22,24 +19,17 @@ public class CellValue {

        StringBuilder sb = new StringBuilder();

-        Iterator<TextBlock> itty = textBlocks.iterator();
-        while (itty.hasNext()) {
-            TextBlock textBlock = itty.next();
-            TextPositionSequence previous = null;
-            for (TextPositionSequence word : textBlock.getSequences()) {
-                if (previous != null) {
-                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
-                        sb.append('\n');
-                    } else {
-                        sb.append(' ');
-                    }
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : textBlock.getSequences()) {
+            if (previous != null) {
+                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
                }
-                sb.append(word.toString());
-                previous = word;
-            }
-            if (itty.hasNext()) {
-                sb.append(' ');
            }
+            sb.append(word.toString());
+            previous = word;
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
@@ -1,33 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.Getter;

@Data
+@AllArgsConstructor
 public class Dictionary {

    public static final String RECOMMENDATION_PREFIX = "recommendation_";

    @Getter
    private List<DictionaryModel> dictionaryModels;
-    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
+    private Map<String, DictionaryModel> localAccessMap;

    @Getter
    private long version;


-    public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
-        this.dictionaryModels = dictionaryModels;
-        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
-        this.version = dictionaryVersion;
-    }
-
-
    public boolean isRecommendation(String type) {

        DictionaryModel model = localAccessMap.get(type);
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;


-import java.io.Serializable;
 import java.util.Set;

 import lombok.AllArgsConstructor;
@@ -9,7 +8,7 @@ import lombok.Data;

@Data
@AllArgsConstructor
-public class DictionaryModel implements Serializable {
+public class DictionaryModel {

    private String type;
    private int rank;
@@ -12,7 +12,7 @@ import lombok.EqualsAndHashCode;
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {

-
+    @EqualsAndHashCode.Include
    private final String word;
    private final String type;
    private boolean redaction;
@@ -20,10 +20,7 @@ public class Entity {
    private String legalBasis;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
    private List<TextPositionSequence> targetSequences;
-
-    @EqualsAndHashCode.Include
    private Integer start;
-    @EqualsAndHashCode.Include
    private Integer end;

    @EqualsAndHashCode.Include
@@ -33,13 +30,8 @@ public class Entity {
    @EqualsAndHashCode.Include
    private int sectionNumber;

-    private boolean isDictionaryEntry;

-    private String textBefore;
-    private String textAfter;
-
-
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {

        this.word = word;
        this.type = type;
@@ -50,13 +42,10 @@ public class Entity {
        this.matchedRule = matchedRule;
        this.sectionNumber = sectionNumber;
        this.legalBasis = legalBasis;
-        this.isDictionaryEntry = isDictionaryEntry;
-        this.textBefore = textBefore;
-        this.textAfter = textAfter;
    }


-    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {

        this.word = word;
        this.type = type;
@@ -64,7 +53,6 @@ public class Entity {
        this.end = end;
        this.headline = headline;
        this.sectionNumber = sectionNumber;
-        this.isDictionaryEntry = isDictionaryEntry;
    }

 }
@@ -51,7 +51,8 @@ public class Section {
        String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");

        return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
-                .toString()
+                .getTextBlock()
+                .getText()
                .equals(value);
    }

@@ -188,13 +189,7 @@ public class Section {
            for (String value : values) {
                if (StringUtils.isNotBlank(value)) {
                    String[] lines = value.split("\n");
-
                    for (String line : lines) {
-
-                        if (line.trim().length() <= 2) {
-                            return;
-                        }
-
                        Set<Entity> found = findEntities(line.trim(), asType, false);

                        // HashSet keeps the older value, but we want the new only.
@@ -222,10 +217,6 @@ public class Section {

    private Set<Entity> findEntities(String value, String asType, boolean caseinsensitive) {

-        if (value.trim().length() <= 2) {
-            return new HashSet<>();
-        }
-
        Set<Entity> found = new HashSet<>();

        String text = caseinsensitive ? searchText.toLowerCase() : searchText;
@@ -239,7 +230,7 @@ public class Section {

            if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(text.charAt(startIndex - 1)) || isSeparator(text
                    .charAt(startIndex - 1))) && (stopIndex == text.length() || isSeparator(text.charAt(stopIndex)))) {
-                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber, false));
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
            }
        } while (startIndex > -1);

@@ -300,12 +291,11 @@ public class Section {
        } else {
            String word = value.toString();

-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
+            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber);
            entity.setRedaction(redact);
            entity.setMatchedRule(ruleNumber);
            entity.setRedactionReason(reason);
-            entity.setTargetSequences(value.getTextBlocks()
-                    .get(0)
+            entity.setTargetSequences(value.getTextBlock()
                    .getSequences()); // Make sure no other cells with same content are highlighted
            entity.setLegalBasis(legalBasis);

@@ -11,7 +11,6 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.configuration.v1.api.model.Colors;
@@ -169,14 +168,8 @@ public class DictionaryService {
    }


-    public Dictionary getDeepCopyDictionary(){
-        List<DictionaryModel> copy = new ArrayList<>();
-
-        dictionary.forEach(dm -> {
-            copy.add(SerializationUtils.clone(dm));
-        });
-
-        return new Dictionary(copy, dictionaryVersion);
+    public Dictionary getThreadSafeDictionary(){
+        return new Dictionary(new ArrayList<>(dictionary), new HashMap<>(localAccessMap), dictionaryVersion);
    }

 }
@@ -32,16 +32,13 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

 import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
 public class EntityRedactionService {

    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;


    public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
@@ -50,7 +47,7 @@ public class EntityRedactionService {
        droolsExecutionService.updateRules();
        long rulesVersion = droolsExecutionService.getRulesVersion();

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary();
+        Dictionary dictionary = dictionaryService.getThreadSafeDictionary();

        Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null));

@@ -84,7 +81,7 @@ public class EntityRedactionService {
                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
+                                .getLegalBasis()));
            }
        }

@@ -95,8 +92,9 @@ public class EntityRedactionService {
    }


-    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, Dictionary dictionary,
-                                     boolean local, Map<Integer, Set<Entity>> hintsPerSectionNumber) {
+    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions,
+                                     Dictionary dictionary, boolean local,
+                                     Map<Integer, Set<Entity>> hintsPerSectionNumber) {

        Set<Entity> documentEntities = new HashSet<>();
        int sectionNumber = 1;
@@ -113,7 +111,6 @@ public class EntityRedactionService {
                    SearchableText searchableRow = new SearchableText();
                    Map<String, CellValue> tabularData = new HashMap<>();
                    int start = 0;
-                    List<Integer> cellStarts = new ArrayList<>();
                    for (Cell cell : row) {
                        if (!singleCellTable && cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
                            continue;
@@ -127,19 +124,14 @@ public class EntityRedactionService {
                                    .replaceAll("\n", "")
                                    .replaceAll(" ", "")
                                    .replaceAll("-", "");
-                            tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
+                            tabularData.put(headerName, new CellValue(cell.getTextBlocks().get(0), cellStart));
                        });
-
+                        start = start + cell.toString().length() + 1; // include automatically appended white space
                        for (TextBlock textBlock : cell.getTextBlocks()) {
-                            // TODO avoid cell overlap merging.
                            searchableRow.addAll(textBlock.getSequences());
                        }
-                        cellStarts.add(cellStart);
-                        start = start + cell.toString().trim().length() + 1;
                    }
-                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary
-                            .getDictionaryModels(), local);
-                    surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);

                    sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                            .isLocal(local)
@@ -161,7 +153,6 @@ public class EntityRedactionService {

            addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
-            surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);

            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                    .isLocal(local)
@@ -183,28 +174,20 @@ public class EntityRedactionService {
            documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair.getSearchableText(), dictionary));

            analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-                if (dictionary.isRecommendation(key)) {
+                if (dictionary.isRecommendation(key)){
                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-                        if (!dictionary.containsValue(key, value)) {
+                        if (!dictionary.containsValue(key, value)){
                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                        }
                    });
                } else {
-                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-
-                        if (dictionary.getLocalAccessMap().get(key) == null) {
-                            log.warn("Dictionary {} is null", key);
-                        }
-
-                        if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                            log.warn("Dictionary {} localEntries is null", key);
-                        }
-
-                        dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
+                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                    });
                }
            });

+
        });

        return documentEntities;
@@ -238,9 +221,9 @@ public class EntityRedactionService {
        String lowercaseInputString = searchableString.toLowerCase();
        for (DictionaryModel model : dictionary) {
            if (model.isCaseInsensitive()) {
-                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
            } else {
-                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
+                found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
            }
        }
        removeEntitiesContainedInLarger(found);
@@ -249,17 +232,10 @@ public class EntityRedactionService {
    }


-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                             boolean local) {
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {

        Set<Entity> found = new HashSet<>();
-
        for (String value : values) {
-
-            if (value.trim().length() <= 2) {
-                continue;
-            }
-
            int startIndex;
            int stopIndex = 0;
            do {
@@ -268,7 +244,7 @@ public class EntityRedactionService {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
                }
            } while (startIndex > -1);
        }
@@ -1,140 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import java.util.List;
-import java.util.Set;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class SurroundingWordsService {
-
-    private final RedactionServiceSettings redactionServiceSettings;
-
-
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
-
-        if (entities.isEmpty()) {
-            return;
-        }
-
-        try {
-            for (Entity entity : entities) {
-
-                if (dictionary.isHint(entity.getType())) {
-                    continue;
-                }
-                findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
-            }
-        } catch (Exception e) {
-            log.warn("Could not get surrounding text!");
-        }
-    }
-
-
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
-                                   List<Integer> cellstarts) {
-
-        if (entities.isEmpty()) {
-            return;
-        }
-
-        try {
-            String searchableString = searchableText.toString();
-
-            if (cellstarts != null) {
-                for (int i = 0; i < cellstarts.size(); i++) {
-
-                    int startOffset = cellstarts.get(i);
-                    int endOffset = -1;
-
-                    if (i + 1 < cellstarts.size()) {
-                        endOffset = cellstarts.get(i + 1);
-                    } else {
-                        endOffset = searchableString.length() - 1;
-                    }
-
-                    String text = searchableString.substring(startOffset, endOffset);
-                    for (Entity entity : entities) {
-
-                        if (dictionary.isHint(entity.getType())) {
-                            continue;
-                        }
-
-                        if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
-                            int entityStartOffset = entity.getStart() - startOffset;
-                            int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
-                        }
-                    }
-                }
-            }
-        } catch (Exception e) {
-            log.warn("Could not get surrounding text!");
-        }
-    }
-
-
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
-
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
-                .getSurroundingWordsOffsetWindow();
-        String textBefore = text.substring(offsetBefore, entityStartOffset);
-        if (!textBefore.isBlank()) {
-            String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
-                    .getNumberOfSurroundingWords() : wordsBefore.length;
-            if (wordsBefore.length > 0) {
-                entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore));
-            }
-        }
-
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
-                .length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
-        String textAfter = text.substring(entityEndOffset, endOffset);
-        if (!textAfter.isBlank()) {
-            String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
-                    .getNumberOfSurroundingWords() : wordsAfter.length;
-            if (wordsAfter.length > 0) {
-                entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter));
-            }
-        }
-    }
-
-
-    private String concatWordsBefore(String[] words, int number) {
-
-        StringBuilder sb = new StringBuilder();
-
-        int startNumber = words.length > number ? words.length - number : 0;
-
-        for (int i = startNumber; i < words.length; i++) {
-            sb.append(words[i]).append(" ");
-        }
-
-        return sb.toString().trim();
-    }
-
-
-    private String concatWordsAfter(String[] words, int number) {
-
-        StringBuilder sb = new StringBuilder();
-
-        for (int i = 0; i < number; i++) {
-            sb.append(words[i]).append(" ");
-        }
-
-        return sb.toString().trim();
-    }
-
-}
@@ -11,7 +11,7 @@ public class TextNormalizationUtilities {
     * @return Text without line-break hyphenation.
     */
    public static String removeHyphenLineBreaks(String text) {
-        return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
    }

 }
@@ -7,9 +7,12 @@ import lombok.Data;
@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
-    
-    private int numberOfSurroundingWords = 3;

-    private int surroundingWordsOffsetWindow = 100;
+    /**
+     * Tenant used in single tenant mode.
+     */
+    private String  defaultTenant = "iqser-id";
+
+    private int flattenImageDpi = 100;

 }
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;

 import java.awt.geom.Point2D;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -42,13 +41,8 @@ public class Cell extends Rectangle {

        StringBuilder sb = new StringBuilder();

-
-        Iterator<TextBlock> itty = textBlocks.iterator();
-        TextPositionSequence previous = null;
-        while (itty.hasNext()) {
-
-            TextBlock textBlock = itty.next();
-
+        for (TextBlock textBlock : textBlocks) {
+            TextPositionSequence previous = null;
            for (TextPositionSequence word : textBlock.getSequences()) {
                if (previous != null) {
                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
@@ -60,7 +54,6 @@ public class Cell extends Rectangle {
                sb.append(word.toString());
                previous = word;
            }
-
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
@@ -68,8 +61,4 @@ public class Cell extends Rectangle {
                .replaceAll(" {2}", " ");
    }

-
-
-
-
 }
@@ -1,111 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
-
-import java.util.ArrayDeque;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.List;
-
-/**
- * Copied and minimal modified from PDFBox.
- */
-public final class QuickSort {
-
-    private QuickSort() {
-
-    }
-
-
-    private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
-        @Override
-        public int compare(Comparable object1, Comparable object2) {
-
-            return object1.compareTo(object2);
-        }
-    };
-
-
-    /**
-     * Sorts the given list using the given comparator.
-     *
-     * @param <T>  type of the objects to be sorted.
-     * @param list list to be sorted
-     * @param cmp  comparator used to compare the objects within the list
-     */
-    public static <T> void sort(List<T> list, Comparator<? super T> cmp) {
-
-        int size = list.size();
-        if (size < 2) {
-            return;
-        }
-        quicksort(list, cmp);
-    }
-
-
-    /**
-     * Sorts the given list using compareTo as comparator.
-     *
-     * @param <T>  type of the objects to be sorted.
-     * @param list list to be sorted
-     */
-    public static <T extends Comparable> void sort(List<T> list) {
-
-        sort(list, (Comparator<T>) OBJCOMP);
-    }
-
-
-    private static <T> void quicksort(List<T> list, Comparator<? super T> cmp) {
-
-        Deque<Integer> stack = new ArrayDeque<Integer>();
-        stack.push(0);
-        stack.push(list.size());
-        while (!stack.isEmpty()) {
-            int right = stack.pop();
-            int left = stack.pop();
-            if (right - left < 2) {
-                continue;
-            }
-            int p = left + ((right - left) / 2);
-            p = partition(list, cmp, p, left, right);
-
-            stack.push(p + 1);
-            stack.push(right);
-
-            stack.push(left);
-            stack.push(p);
-        }
-    }
-
-
-    private static <T> int partition(List<T> list, Comparator<? super T> cmp, int p, int start, int end) {
-
-        int l = start;
-        int h = end - 2;
-        T piv = list.get(p);
-        swap(list, p, end - 1);
-
-        while (l < h) {
-            if (cmp.compare(list.get(l), piv) <= 0) {
-                l++;
-            } else if (cmp.compare(piv, list.get(h)) <= 0) {
-                h--;
-            } else {
-                swap(list, l, h);
-            }
-        }
-        int idx = h;
-        if (cmp.compare(list.get(h), piv) < 0) {
-            idx++;
-        }
-        swap(list, end - 1, idx);
-        return idx;
-    }
-
-
-    private static <T> void swap(List<T> list, int i, int j) {
-
-        T tmp = list.get(i);
-        list.set(i, list.get(j));
-        list.set(j, tmp);
-    }
-
-}
@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.utils;

 import java.math.BigDecimal;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;

@@ -12,29 +13,23 @@ public class Utils {

    private final static float EPSILON = 0.1f;

-
    public static boolean feq(double f1, double f2) {
-
        return (Math.abs(f1 - f2) < EPSILON);
    }

-
    public static float round(double d, int decimalPlace) {
-
        BigDecimal bd = BigDecimal.valueOf(d);
        bd = bd.setScale(decimalPlace, BigDecimal.ROUND_HALF_UP);
        return bd.floatValue();
    }

-
-    public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
-
+	public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
        try {
-            QuickSort.sort(list, comparator);
-        } catch (IllegalArgumentException e) {
-            // This should not happen since we use QuickSort from PDFBox
+            Collections.sort(list, comparator);
+        } catch (IllegalArgumentException e){
+            //TODO Figure out why this happens.
            log.warn(e.getMessage());
        }
-    }
+	}

 }
@@ -256,7 +256,6 @@ public class AnnotationHighlightService {
                .manual(true)
                .status(manualRedactionEntry.getStatus())
                .manualRedactionType(ManualRedactionType.ADD)
-                .isDictionaryEntry(false)
                .build();
    }

@@ -275,9 +274,6 @@ public class AnnotationHighlightService {
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
-                .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
                .build();
    }

@@ -0,0 +1,68 @@
+package com.iqser.red.service.redaction.v1.server.visualization.service;
+
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
+import org.apache.pdfbox.rendering.ImageType;
+import org.apache.pdfbox.rendering.PDFRenderer;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class PdfFlattenService {
+
+    private final RedactionServiceSettings settings;
+
+    public PDDocument flattenPDF(PDDocument sourceDoc) throws IOException {
+
+        PDDocument destDoc = new PDDocument();
+
+        PDFRenderer pdfRenderer = new PDFRenderer(sourceDoc);
+
+        final int pageCount = sourceDoc.getDocumentCatalog().getPages().getCount();
+
+        log.info(pageCount + " page" + (pageCount == 1 ? "" : "s") + " to flatten.");
+
+        for (int i = 0; i < pageCount; i += 1) {
+
+            log.info("Flattening page " + (i + 1) + " of " + pageCount + "...");
+
+            BufferedImage img = pdfRenderer.renderImageWithDPI(i, settings.getFlattenImageDpi(), ImageType.RGB);
+
+            log.info("Image rendered in memory (" + img.getWidth() + "x" + img.getHeight() + " " + settings.getFlattenImageDpi() + "DPI).  Adding to PDF...");
+
+            PDPage imagePage = new PDPage(new PDRectangle(img.getWidth(), img.getHeight()));
+            destDoc.addPage(imagePage);
+
+            PDImageXObject imgObj = LosslessFactory.createFromImage(destDoc, img);
+
+            PDPageContentStream imagePageContentStream = new PDPageContentStream(destDoc, imagePage);
+            imagePageContentStream.drawImage(imgObj, 0, 0);
+
+            log.info("Image added successfully.");
+
+            imagePageContentStream.close();
+
+            img.flush();
+        }
+
+        log.info("New flattened PDF created in memory.");
+
+        sourceDoc.close();
+
+        return destDoc;
+    }
+
+}
@@ -331,7 +331,6 @@ public class RedactionIntegrationTest {
    @Test
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

-        long start = System.currentTimeMillis();
        System.out.println("noExceptionShouldBeThrownForAnyFiles");
        ClassLoader loader = getClass().getClassLoader();
        URL url = loader.getResource("files");
@@ -357,10 +356,6 @@ public class RedactionIntegrationTest {
            });
        }

-        long end = System.currentTimeMillis();
-
-        System.out.println("duration: " + (end - start));
-
    }


@@ -387,7 +382,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -396,11 +391,11 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.redact(request);

-//        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
-//            if(!entry.isHint()){
-//                System.out.println(entry.getPositions().get(0).getPage() +":"+ entry.getTextBefore() +"--->"+ entry.getValue() + "--->" + entry.getTextAfter());
-//            }
-//        });
+        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
+            if(entry.isRecommendation()){
+                System.out.println(entry.getValue());
+            }
+        });

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
            fileOutputStream.write(result.getDocument());
@@ -104,8 +104,8 @@ public class EntityRedactionServiceTest {
    public void testNestedEntitiesRemoval() {

        Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
        entities.add(nested);
        entities.add(nesting);
        entityRedactionService.removeEntitiesContainedInLarger(entities);
@@ -404,7 +404,7 @@ public class EntityRedactionServiceTest {
            entityRedactionService.processDocument(classifiedDoc, null);
            assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
-            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
+            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(8);
        }
    }

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TextNormalizationUtilities {
+
+    /**
+     * Revert hyphenation due to line breaks.
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+    }
+
+}
@@ -10,11 +10,11 @@ public class TextNormalizationUtilitiesTest {

        String test = "Without these peo-\nple, this conference would not happen";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("people");
+                .contains("\npeople");

        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("einzelnen", "Vertrages");
+                .contains("\neinzelnen", "\nVertrages");

    }