Fixed 'Comparison method violates its general contract' by using QuickSort from PDFBox

Pull request #83 : RED-882: Added textBefore and textAfter to redaction log
Merge in RED/redaction-service from RED-882 to master * commit 'caf6277de9e0d175d3ff11c1e7819cfbc37f3539': RED-882: Added textBefore and textAfter to redaction log
2020-12-22 16:04:29 +01:00 · 2020-12-18 14:41:23 +01:00 · 2020-12-18 14:31:27 +01:00 · 2020-12-11 11:39:00 +01:00 · 2020-12-11 11:26:48 +01:00
17 changed files with 387 additions and 150 deletions
@@ -34,4 +34,7 @@ public class RedactionLogEntry {
    private ManualRedactionType manualRedactionType;
    private boolean isDictionaryEntry;

+    private String textBefore;
+    private String textAfter;
+
 }
@@ -5,15 +5,17 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

-import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;

 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@Service
@RequiredArgsConstructor
 public class ClassificationService {
@@ -28,7 +30,7 @@ public class ClassificationService {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

-        System.out.println(document.getFontSizeCounter().getCountPerValue());
+        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (Page page : document.getPages()) {
            Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
@@ -39,6 +41,7 @@ public class ClassificationService {


    public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
+
        for (AbstractTextContainer textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextBlock) {
                classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
@@ -47,24 +50,33 @@ public class ClassificationService {
    }


-    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
+    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
+                              List<Float> headlineFontSizes) {
+
        if (document.getFontSizeCounter().getMostPopular() == null) {
            // TODO Figure out why this happens.
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification("Header");

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification("Footer");
-        }  else if (page.getPageNumber() == 1
-                && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
-                && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
+                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
+                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
+                .getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                textBlock.setClassification("Title");
            }
-        }
-        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
+                .getFontSizeCounter()
+                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
+                .equals("bold")) {

            for (int i = 1; i <= headlineFontSizes.size(); i++) {
                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
@@ -72,20 +84,34 @@ public class ClassificationService {
                    document.setHeadlines(true);
                }
            }
-        }else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
+                .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
+                .equals("bold") && !document.getFontStyleCounter()
+                .getMostPopular()
+                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
            document.setHeadlines(true);
-        }
-        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+                .getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle()
+                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
            textBlock.setClassification("TextBlock Bold");
-        }
-        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
+                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
+                .equals(document.getFontStyleCounter()
+                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+                .getMostPopular()) {
            textBlock.setClassification("TextBlock");
-        }
-        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+                .getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle()
+                .equals("italic") && !document.getFontStyleCounter()
+                .getMostPopular()
+                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification("TextBlock Italic");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
            textBlock.setClassification("TextBlock Unknown");
        }
    }
+
 }
@@ -24,11 +24,12 @@ import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationSer
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;

 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@RestController
@RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
@@ -37,7 +38,6 @@ public class RedactionController implements RedactionResource {
    private final PdfSegmentationService pdfSegmentationService;
    private final AnnotationHighlightService annotationHighlightService;
    private final EntityRedactionService entityRedactionService;
-    private final PdfFlattenService pdfFlattenService;
    private final DroolsExecutionService droolsExecutionService;


@@ -48,15 +48,14 @@ public class RedactionController implements RedactionResource {
            pdDocument.setAllSecurityToBeRemoved(true);

            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+
+            log.info("Document structure analysis successful, starting redaction analysis...");
+
            entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
            annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
                    .getManualRedactions());

-            if (redactionRequest.isFlatRedaction()) {
-                PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
-                return convert(flatDocument, classifiedDoc.getPages()
-                        .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
-            }
+            log.info("Redaction analysis successful...");

            return convert(pdDocument, classifiedDoc.getPages()
                    .size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
@@ -35,8 +35,11 @@ public class Entity {

    private boolean isDictionaryEntry;

+    private String textBefore;
+    private String textAfter;

-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry) {
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {

        this.word = word;
        this.type = type;
@@ -48,6 +51,8 @@ public class Entity {
        this.sectionNumber = sectionNumber;
        this.legalBasis = legalBasis;
        this.isDictionaryEntry = isDictionaryEntry;
+        this.textBefore = textBefore;
+        this.textAfter = textAfter;
    }


@@ -188,7 +188,13 @@ public class Section {
            for (String value : values) {
                if (StringUtils.isNotBlank(value)) {
                    String[] lines = value.split("\n");
+
                    for (String line : lines) {
+
+                        if (line.trim().length() <= 2) {
+                            return;
+                        }
+
                        Set<Entity> found = findEntities(line.trim(), asType, false);

                        // HashSet keeps the older value, but we want the new only.
@@ -216,6 +222,10 @@ public class Section {

    private Set<Entity> findEntities(String value, String asType, boolean caseinsensitive) {

+        if (value.trim().length() <= 2) {
+            return new HashSet<>();
+        }
+
        Set<Entity> found = new HashSet<>();

        String text = caseinsensitive ? searchText.toLowerCase() : searchText;
@@ -294,7 +304,8 @@ public class Section {
            entity.setRedaction(redact);
            entity.setMatchedRule(ruleNumber);
            entity.setRedactionReason(reason);
-            entity.setTargetSequences(value.getTextBlocks().get(0)
+            entity.setTargetSequences(value.getTextBlocks()
+                    .get(0)
                    .getSequences()); // Make sure no other cells with same content are highlighted
            entity.setLegalBasis(legalBasis);

@@ -41,6 +41,7 @@ public class EntityRedactionService {

    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;
+    private final SurroundingWordsService surroundingWordsService;


    public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
@@ -83,7 +84,7 @@ public class EntityRedactionService {
                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry()));
+                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
            }
        }

@@ -94,9 +95,8 @@ public class EntityRedactionService {
    }


-    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions,
-                                     Dictionary dictionary, boolean local,
-                                     Map<Integer, Set<Entity>> hintsPerSectionNumber) {
+    private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, Dictionary dictionary,
+                                     boolean local, Map<Integer, Set<Entity>> hintsPerSectionNumber) {

        Set<Entity> documentEntities = new HashSet<>();
        int sectionNumber = 1;
@@ -113,6 +113,7 @@ public class EntityRedactionService {
                    SearchableText searchableRow = new SearchableText();
                    Map<String, CellValue> tabularData = new HashMap<>();
                    int start = 0;
+                    List<Integer> cellStarts = new ArrayList<>();
                    for (Cell cell : row) {
                        if (!singleCellTable && cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
                            continue;
@@ -128,12 +129,17 @@ public class EntityRedactionService {
                                    .replaceAll("-", "");
                            tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
                        });
-                        start = start + cell.toString().length() + 1;
+
                        for (TextBlock textBlock : cell.getTextBlocks()) {
+                            // TODO avoid cell overlap merging.
                            searchableRow.addAll(textBlock.getSequences());
                        }
+                        cellStarts.add(cellStart);
+                        start = start + cell.toString().trim().length() + 1;
                    }
-                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary
+                            .getDictionaryModels(), local);
+                    surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);

                    sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                            .isLocal(local)
@@ -155,6 +161,7 @@ public class EntityRedactionService {

            addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
+            surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);

            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                    .isLocal(local)
@@ -176,20 +183,20 @@ public class EntityRedactionService {
            documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair.getSearchableText(), dictionary));

            analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-                if (dictionary.isRecommendation(key)){
+                if (dictionary.isRecommendation(key)) {
                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-                        if (!dictionary.containsValue(key, value)){
+                        if (!dictionary.containsValue(key, value)) {
                            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
                        }
                    });
                } else {
-                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
+                    analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {

-                        if(dictionary.getLocalAccessMap().get(key) == null){
+                        if (dictionary.getLocalAccessMap().get(key) == null) {
                            log.warn("Dictionary {} is null", key);
                        }

-                        if(dictionary.getLocalAccessMap().get(key).getLocalEntries() == null){
+                        if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
                            log.warn("Dictionary {} localEntries is null", key);
                        }

@@ -198,7 +205,6 @@ public class EntityRedactionService {
                }
            });

-
        });

        return documentEntities;
@@ -243,10 +249,17 @@ public class EntityRedactionService {
    }


-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber, boolean local) {
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
+                             boolean local) {

        Set<Entity> found = new HashSet<>();
+
        for (String value : values) {
+
+            if (value.trim().length() <= 2) {
+                continue;
+            }
+
            int startIndex;
            int stopIndex = 0;
            do {
@@ -0,0 +1,140 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class SurroundingWordsService {
+
+    private final RedactionServiceSettings redactionServiceSettings;
+
+
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
+
+        if (entities.isEmpty()) {
+            return;
+        }
+
+        try {
+            for (Entity entity : entities) {
+
+                if (dictionary.isHint(entity.getType())) {
+                    continue;
+                }
+                findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
+            }
+        } catch (Exception e) {
+            log.warn("Could not get surrounding text!");
+        }
+    }
+
+
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
+                                   List<Integer> cellstarts) {
+
+        if (entities.isEmpty()) {
+            return;
+        }
+
+        try {
+            String searchableString = searchableText.toString();
+
+            if (cellstarts != null) {
+                for (int i = 0; i < cellstarts.size(); i++) {
+
+                    int startOffset = cellstarts.get(i);
+                    int endOffset = -1;
+
+                    if (i + 1 < cellstarts.size()) {
+                        endOffset = cellstarts.get(i + 1);
+                    } else {
+                        endOffset = searchableString.length() - 1;
+                    }
+
+                    String text = searchableString.substring(startOffset, endOffset);
+                    for (Entity entity : entities) {
+
+                        if (dictionary.isHint(entity.getType())) {
+                            continue;
+                        }
+
+                        if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
+                            int entityStartOffset = entity.getStart() - startOffset;
+                            int entityEndOffset = entity.getEnd() - startOffset;
+                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.warn("Could not get surrounding text!");
+        }
+    }
+
+
+    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
+
+        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
+                .getSurroundingWordsOffsetWindow();
+        String textBefore = text.substring(offsetBefore, entityStartOffset);
+        if (!textBefore.isBlank()) {
+            String[] wordsBefore = textBefore.split(" ");
+            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
+                    .getNumberOfSurroundingWords() : wordsBefore.length;
+            if (wordsBefore.length > 0) {
+                entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore));
+            }
+        }
+
+        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
+                .length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        String textAfter = text.substring(entityEndOffset, endOffset);
+        if (!textAfter.isBlank()) {
+            String[] wordsAfter = textAfter.split(" ");
+            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
+                    .getNumberOfSurroundingWords() : wordsAfter.length;
+            if (wordsAfter.length > 0) {
+                entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter));
+            }
+        }
+    }
+
+
+    private String concatWordsBefore(String[] words, int number) {
+
+        StringBuilder sb = new StringBuilder();
+
+        int startNumber = words.length > number ? words.length - number : 0;
+
+        for (int i = startNumber; i < words.length; i++) {
+            sb.append(words[i]).append(" ");
+        }
+
+        return sb.toString().trim();
+    }
+
+
+    private String concatWordsAfter(String[] words, int number) {
+
+        StringBuilder sb = new StringBuilder();
+
+        for (int i = 0; i < number; i++) {
+            sb.append(words[i]).append(" ");
+        }
+
+        return sb.toString().trim();
+    }
+
+}
@@ -11,7 +11,7 @@ public class TextNormalizationUtilities {
     * @return Text without line-break hyphenation.
     */
    public static String removeHyphenLineBreaks(String text) {
-        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+        return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
    }

 }
@@ -7,12 +7,9 @@ import lombok.Data;
@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
+    
+    private int numberOfSurroundingWords = 3;

-    /**
-     * Tenant used in single tenant mode.
-     */
-    private String  defaultTenant = "iqser-id";
-
-    private int flattenImageDpi = 100;
+    private int surroundingWordsOffsetWindow = 100;

 }
@@ -42,10 +42,13 @@ public class Cell extends Rectangle {

        StringBuilder sb = new StringBuilder();

+
        Iterator<TextBlock> itty = textBlocks.iterator();
+        TextPositionSequence previous = null;
        while (itty.hasNext()) {
+
            TextBlock textBlock = itty.next();
-            TextPositionSequence previous = null;
+
            for (TextPositionSequence word : textBlock.getSequences()) {
                if (previous != null) {
                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
@@ -57,9 +60,7 @@ public class Cell extends Rectangle {
                sb.append(word.toString());
                previous = word;
            }
-            if (itty.hasNext()) {
-                sb.append(' ');
-            }
+
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
@@ -67,4 +68,8 @@ public class Cell extends Rectangle {
                .replaceAll(" {2}", " ");
    }

+
+
+
+
 }
@@ -0,0 +1,111 @@
+package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
+
+import java.util.ArrayDeque;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.List;
+
+/**
+ * Copied and minimal modified from PDFBox.
+ */
+public final class QuickSort {
+
+    private QuickSort() {
+
+    }
+
+
+    private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
+        @Override
+        public int compare(Comparable object1, Comparable object2) {
+
+            return object1.compareTo(object2);
+        }
+    };
+
+
+    /**
+     * Sorts the given list using the given comparator.
+     *
+     * @param <T>  type of the objects to be sorted.
+     * @param list list to be sorted
+     * @param cmp  comparator used to compare the objects within the list
+     */
+    public static <T> void sort(List<T> list, Comparator<? super T> cmp) {
+
+        int size = list.size();
+        if (size < 2) {
+            return;
+        }
+        quicksort(list, cmp);
+    }
+
+
+    /**
+     * Sorts the given list using compareTo as comparator.
+     *
+     * @param <T>  type of the objects to be sorted.
+     * @param list list to be sorted
+     */
+    public static <T extends Comparable> void sort(List<T> list) {
+
+        sort(list, (Comparator<T>) OBJCOMP);
+    }
+
+
+    private static <T> void quicksort(List<T> list, Comparator<? super T> cmp) {
+
+        Deque<Integer> stack = new ArrayDeque<Integer>();
+        stack.push(0);
+        stack.push(list.size());
+        while (!stack.isEmpty()) {
+            int right = stack.pop();
+            int left = stack.pop();
+            if (right - left < 2) {
+                continue;
+            }
+            int p = left + ((right - left) / 2);
+            p = partition(list, cmp, p, left, right);
+
+            stack.push(p + 1);
+            stack.push(right);
+
+            stack.push(left);
+            stack.push(p);
+        }
+    }
+
+
+    private static <T> int partition(List<T> list, Comparator<? super T> cmp, int p, int start, int end) {
+
+        int l = start;
+        int h = end - 2;
+        T piv = list.get(p);
+        swap(list, p, end - 1);
+
+        while (l < h) {
+            if (cmp.compare(list.get(l), piv) <= 0) {
+                l++;
+            } else if (cmp.compare(piv, list.get(h)) <= 0) {
+                h--;
+            } else {
+                swap(list, l, h);
+            }
+        }
+        int idx = h;
+        if (cmp.compare(list.get(h), piv) < 0) {
+            idx++;
+        }
+        swap(list, end - 1, idx);
+        return idx;
+    }
+
+
+    private static <T> void swap(List<T> list, int i, int j) {
+
+        T tmp = list.get(i);
+        list.set(i, list.get(j));
+        list.set(j, tmp);
+    }
+
+}
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.utils;

 import java.math.BigDecimal;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;

@@ -13,23 +12,29 @@ public class Utils {

    private final static float EPSILON = 0.1f;

+
    public static boolean feq(double f1, double f2) {
+
        return (Math.abs(f1 - f2) < EPSILON);
    }

+
    public static float round(double d, int decimalPlace) {
+
        BigDecimal bd = BigDecimal.valueOf(d);
        bd = bd.setScale(decimalPlace, BigDecimal.ROUND_HALF_UP);
        return bd.floatValue();
    }

-	public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
+
+    public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
+
        try {
-            Collections.sort(list, comparator);
-        } catch (IllegalArgumentException e){
-            //TODO Figure out why this happens.
+            QuickSort.sort(list, comparator);
+        } catch (IllegalArgumentException e) {
+            // This should not happen since we use QuickSort from PDFBox
            log.warn(e.getMessage());
        }
-	}
+    }

 }
@@ -276,6 +276,8 @@ public class AnnotationHighlightService {
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
                .isDictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
                .build();
    }

@@ -1,68 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.visualization.service;
-
-import java.awt.image.BufferedImage;
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
-import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
-import org.apache.pdfbox.rendering.ImageType;
-import org.apache.pdfbox.rendering.PDFRenderer;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfFlattenService {
-
-    private final RedactionServiceSettings settings;
-
-    public PDDocument flattenPDF(PDDocument sourceDoc) throws IOException {
-
-        PDDocument destDoc = new PDDocument();
-
-        PDFRenderer pdfRenderer = new PDFRenderer(sourceDoc);
-
-        final int pageCount = sourceDoc.getDocumentCatalog().getPages().getCount();
-
-        log.info(pageCount + " page" + (pageCount == 1 ? "" : "s") + " to flatten.");
-
-        for (int i = 0; i < pageCount; i += 1) {
-
-            log.info("Flattening page " + (i + 1) + " of " + pageCount + "...");
-
-            BufferedImage img = pdfRenderer.renderImageWithDPI(i, settings.getFlattenImageDpi(), ImageType.RGB);
-
-            log.info("Image rendered in memory (" + img.getWidth() + "x" + img.getHeight() + " " + settings.getFlattenImageDpi() + "DPI).  Adding to PDF...");
-
-            PDPage imagePage = new PDPage(new PDRectangle(img.getWidth(), img.getHeight()));
-            destDoc.addPage(imagePage);
-
-            PDImageXObject imgObj = LosslessFactory.createFromImage(destDoc, img);
-
-            PDPageContentStream imagePageContentStream = new PDPageContentStream(destDoc, imagePage);
-            imagePageContentStream.drawImage(imgObj, 0, 0);
-
-            log.info("Image added successfully.");
-
-            imagePageContentStream.close();
-
-            img.flush();
-        }
-
-        log.info("New flattened PDF created in memory.");
-
-        sourceDoc.close();
-
-        return destDoc;
-    }
-
-}
@@ -331,6 +331,7 @@ public class RedactionIntegrationTest {
    @Test
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

+        long start = System.currentTimeMillis();
        System.out.println("noExceptionShouldBeThrownForAnyFiles");
        ClassLoader loader = getClass().getClassLoader();
        URL url = loader.getResource("files");
@@ -356,6 +357,10 @@ public class RedactionIntegrationTest {
            });
        }

+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+
    }


@@ -382,7 +387,7 @@ public class RedactionIntegrationTest {

        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -391,11 +396,11 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.redact(request);

-        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
-            if(entry.isDictionaryEntry()){
-                System.out.println(entry.getValue());
-            }
-        });
+//        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
+//            if(!entry.isHint()){
+//                System.out.println(entry.getPositions().get(0).getPage() +":"+ entry.getTextBefore() +"--->"+ entry.getValue() + "--->" + entry.getTextAfter());
+//            }
+//        });

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
            fileOutputStream.write(result.getDocument());
@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.utils;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class TextNormalizationUtilities {
-
-    /**
-     * Revert hyphenation due to line breaks.
-     * @param text Text to be processed.
-     * @return Text without line-break hyphenation.
-     */
-    public static String removeHyphenLineBreaks(String text) {
-        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
-    }
-
-}
@@ -10,11 +10,11 @@ public class TextNormalizationUtilitiesTest {

        String test = "Without these peo-\nple, this conference would not happen";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("\npeople");
+                .contains("people");

        test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
        Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
-                .contains("\neinzelnen", "\nVertrages");
+                .contains("einzelnen", "Vertrages");

    }
Author	SHA1	Message	Date
Dominique Eifländer	000b145e71	Fixed 'Comparison method violates its general contract' by using QuickSort from PDFBox	2020-12-22 16:04:29 +01:00
Dominique Eiflaender	7dbc735b16	Pull request #83 : RED-882: Added textBefore and textAfter to redaction log Merge in RED/redaction-service from RED-882 to master * commit 'caf6277de9e0d175d3ff11c1e7819cfbc37f3539': RED-882: Added textBefore and textAfter to redaction log	2020-12-18 14:41:23 +01:00
Dominique Eifländer	caf6277de9	RED-882: Added textBefore and textAfter to redaction log	2020-12-18 14:31:27 +01:00
Dominique Eiflaender	add196f913	Pull request #82 : RED-871: Fixed endless processing on document with corrupted contentStream Merge in RED/redaction-service from RED-871 to master * commit 'bfa363a3d2903bb9bf68c7721740e95b2f1a4925': RED-871: Fixed endless processing on document with corrupted contentStream	2020-12-11 11:39:00 +01:00
deiflaender	bfa363a3d2	RED-871: Fixed endless processing on document with corrupted contentStream	2020-12-11 11:26:48 +01:00