Remove redundant comment

RED-101: Add features as requested in PR
- Add rule for non-vertebrate studies - Don't highlight ambiguous cell values - Rewrite rules in more human-readable fashion
2020-08-18 20:53:41 +02:00 · 2020-08-18 20:35:40 +02:00 · 2020-08-18 20:35:17 +02:00 · 2020-08-13 17:48:37 +02:00 · 2020-08-13 16:20:53 +02:00 · 2020-08-13 16:10:22 +02:00
29 changed files with 1079 additions and 503 deletions
@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ManualRedactionEntry {
+
+    private String type;
+    private String value;
+    private String reason;
+    private List<Rectangle> positions = new ArrayList<>();
+
+    private String section;
+    private int sectionNumber;
+
+}
@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ManualRedactions {
+
+    private Set<String> idsToRemove = new HashSet<>();
+    private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
+}
@@ -3,9 +3,11 @@ package com.iqser.red.service.redaction.v1.model;
 import java.util.ArrayList;
 import java.util.List;

+import lombok.Builder;
 import lombok.Data;

@Data
+@Builder
 public class RedactionLogEntry {

    private String id;
@@ -16,6 +18,10 @@ public class RedactionLogEntry {
    private boolean isHint;
    private String section;
    private float[] color;
+
+    @Builder.Default
    private List<Rectangle> positions = new ArrayList<>();
+    private int sectionNumber;
+    private boolean manual;

 }
@@ -13,4 +13,5 @@ public class RedactionRequest {

    private byte[] document;
    private boolean flatRedaction;
+    private ManualRedactions manualRedactions;
 }
@@ -56,6 +56,11 @@
            <artifactId>jts-core</artifactId>
            <version>1.16.1</version>
        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>29.0-jre</version>
+        </dependency>
        <!-- commons -->
        <dependency>
            <groupId>com.iqser.gin4.commons</groupId>
@@ -41,4 +41,16 @@ public class Paragraph {
        return tables;
    }

+
+    public List<TextBlock> getTextBlocks() {
+
+        List<TextBlock> textBlocks = new ArrayList<>();
+        pageBlocks.forEach(block -> {
+            if (block instanceof TextBlock) {
+                textBlocks.add((TextBlock) block);
+            }
+        });
+        return textBlocks;
+    }
+
 }
@@ -44,8 +44,8 @@ public class RedactionController implements RedactionResource {
            pdDocument.setAllSecurityToBeRemoved(true);

            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-            entityRedactionService.processDocument(classifiedDoc);
-            annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction());
+            entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
+            annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest.getManualRedactions());

            if (redactionRequest.isFlatRedaction()) {
                PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
@@ -5,6 +5,9 @@ import java.util.List;

 import org.apache.pdfbox.text.TextPosition;

+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+
 import lombok.Data;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
@@ -22,36 +25,48 @@ public class TextPositionSequence implements CharSequence {

    private final int page;

-    public TextPositionSequence(List<TextPosition> textPositions, int page){
+
+    public TextPositionSequence(List<TextPosition> textPositions, int page) {
+
        this.textPositions = textPositions;
        this.page = page;
    }

+
    @Override
    public int length() {
+
        return textPositions.size();
    }

+
    @Override
    public char charAt(int index) {
+
        TextPosition textPosition = textPositionAt(index);
        String text = textPosition.getUnicode();
        return text.charAt(0);
    }

+
    public char charAt(int index, boolean caseInSensitive) {
+
        TextPosition textPosition = textPositionAt(index);
        String text = textPosition.getUnicode();
        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
    }

+
    @Override
    public TextPositionSequence subSequence(int start, int end) {
+
        return new TextPositionSequence(textPositions.subList(start, end), page);
    }

+
    @Override
    public String toString() {
+
        StringBuilder builder = new StringBuilder(length());
        for (int i = 0; i < length(); i++) {
            builder.append(charAt(i));
@@ -59,15 +74,21 @@ public class TextPositionSequence implements CharSequence {
        return builder.toString();
    }

+
    public TextPosition textPositionAt(int index) {
+
        return textPositions.get(index);
    }

+
    public void add(TextPosition textPosition) {
+
        this.textPositions.add(textPosition);
    }

+
    public float getX1() {
+
        if (textPositions.get(0).getRotation() == 90) {
            return textPositions.get(0).getYDirAdj() - getTextHeight();
        } else {
@@ -75,15 +96,20 @@ public class TextPositionSequence implements CharSequence {
        }
    }

+
    public float getX2() {
+
        if (textPositions.get(0).getRotation() == 90) {
            return textPositions.get(0).getYDirAdj();
        } else {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            return textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
        }
    }

+
    public float getY1() {
+
        if (textPositions.get(0).getRotation() == 90) {
            return textPositions.get(0).getXDirAdj();
        } else {
@@ -91,30 +117,46 @@ public class TextPositionSequence implements CharSequence {
        }
    }

+
    public float getY2() {
+
        if (textPositions.get(0).getRotation() == 90) {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() -2 ;
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
        } else {
            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
        }
    }

+
    public float getTextHeight() {
+
        return textPositions.get(0).getHeightDir() + 2;
    }

+
    public float getHeight() {
+
        return getY2() - getY1();
    }

+
    public float getWidth() {
+
        return getX2() - getX1();
    }

+
    public String getFont() {
-        return textPositions.get(0).getFont().toString().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
+
+        return textPositions.get(0)
+                .getFont()
+                .toString()
+                .toLowerCase()
+                .replaceAll(",bold", "")
+                .replaceAll(",italic", "");
    }

+
    public String getFontStyle() {

        String lowercaseFontName = textPositions.get(0).getFont().toString().toLowerCase();
@@ -131,16 +173,51 @@ public class TextPositionSequence implements CharSequence {

    }

+
    public float getFontSize() {
+
        return textPositions.get(0).getFontSizeInPt();
    }

+
    public float getSpaceWidth() {
+
        return textPositions.get(0).getWidthOfSpace();
    }

+
    public int getRotation() {
+
        return textPositions.get(0).getRotation();
    }

+
+    public Rectangle getRectangle() {
+
+        float height = textPositions.get(0).getHeightDir() + 2;
+
+        float posXInit;
+        float posXEnd;
+        float posYInit;
+        float posYEnd;
+
+        if (textPositions.get(0).getRotation() == 90) {
+
+            posXEnd = textPositions.get(0).getYDirAdj() + 2;
+            posXInit = textPositions.get(0).getYDirAdj() - height;
+            posYInit = textPositions.get(0).getXDirAdj();
+            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
+        } else {
+
+            posXInit = textPositions.get(0).getXDirAdj();
+            posXEnd = textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + 2;
+        }
+
+        return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
+    }
+
 }
@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 import java.util.ArrayList;
 import java.util.List;

+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;

@@ -16,6 +18,7 @@ public class Entity {
    private boolean redaction;
    private String redactionReason;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
+    private List<TextPositionSequence> targetSequences;
    private Integer start;
    private Integer end;

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;

 import java.util.ArrayList;
 import java.util.List;
-import java.util.UUID;

 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;

@@ -20,6 +19,6 @@ public class EntityPositionSequence {
    @EqualsAndHashCode.Exclude
    private List<TextPositionSequence> sequences = new ArrayList<>();
    private int pageNumber;
-    private final UUID id;
+    private final String id;

 }
@@ -1,11 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.UUID;
 import java.util.regex.Pattern;

 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

 public class SearchableText {
@@ -25,9 +26,16 @@ public class SearchableText {
    }


-    @SuppressWarnings("checkstyle:ModifiedControlVariable")
    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {

+        return getSequences(searchString, caseInsensitive, null);
+
+    }
+
+    @SuppressWarnings("checkstyle:ModifiedControlVariable")
+    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
+                                                     List<TextPositionSequence> sequencesSubList) {
+
        String normalizedSearchString;
        if (caseInsensitive) {
            normalizedSearchString = searchString.toLowerCase();
@@ -40,37 +48,50 @@ public class SearchableText {

        List<TextPositionSequence> crossSequenceParts = new ArrayList<>();
        List<EntityPositionSequence> finalMatches = new ArrayList<>();
-        for (int i = 0; i < sequences.size(); i++) {
-            TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
-            for (int j = 0; j < sequences.get(i).length(); j++) {

-                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInsensitive) == ' ' && sequences.get(i - 1)
-                        .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && sequences.get(i)
-                        .charAt(j, caseInsensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInsensitive) == ' ') {
-                    if (j == sequences.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
+        List<TextPositionSequence> searchSpace;
+        if (sequencesSubList != null) {
+            int subListIndex = Collections.indexOfSubList(sequences, sequencesSubList);
+            if (subListIndex != -1) {
+                searchSpace = sequences.subList(subListIndex, subListIndex + sequencesSubList.size());
+            } else {
+                searchSpace = sequences;
+            }
+        } else {
+            searchSpace = sequences;
+        }
+
+        for (int i = 0; i < searchSpace.size(); i++) {
+            TextPositionSequence partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
+            for (int j = 0; j < searchSpace.get(i).length(); j++) {
+
+                if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
+                        .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
                        crossSequenceParts.add(partMatch);
                    }
                    continue;
                }

-                if (j == 0 && sequences.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && sequences.get(i - 1)
-                        .charAt(sequences.get(i - 1)
+                if (j == 0 && searchSpace.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && searchSpace.get(i - 1)
+                        .charAt(searchSpace.get(i - 1)
                                .length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
                    counter++;
                }

-                if (sequences.get(i)
-                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
+                if (searchSpace.get(i)
+                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
                        .charAt(j, caseInsensitive) == '-') {

-                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i)
-                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
-                            .charAt(sequences.get(i - 1)
-                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
-                            .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) != ' ' && sequences.get(i)
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(searchSpace.get(i - 1)
+                            .charAt(searchSpace.get(i - 1)
+                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
+                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
                            .charAt(j, caseInsensitive) != ' ') {
-                        partMatch.add(sequences.get(i).textPositionAt(j));
-                        if (!(j == sequences.get(i).length() - 1 && sequences.get(i)
+                        partMatch.add(searchSpace.get(i).textPositionAt(j));
+                        if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                            counter++;
                        }
@@ -79,19 +100,19 @@ public class SearchableText {
                    if (counter == searchString.length()) {
                        crossSequenceParts.add(partMatch);

-                        if (i == sequences.size() - 1 && j == sequences.get(i).length() - 1 || j != sequences.get(i)
-                                .length() - 1 && isSeparator(sequences.get(i)
-                                .charAt(j + 1, caseInsensitive)) || j == sequences.get(i)
-                                .length() - 1 && isSeparator(sequences.get(i + 1)
-                                .charAt(0, caseInsensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
-                                .charAt(j, caseInsensitive) != ' ' && sequences.get(i + 1)
+                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
+                                .length() - 1 && isSeparator(searchSpace.get(i)
+                                .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
+                                .length() - 1 && isSeparator(searchSpace.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
                                .charAt(0, caseInsensitive) != ' ') {
                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
                        }

                        counter = 0;
                        crossSequenceParts = new ArrayList<>();
-                        partMatch = new TextPositionSequence(sequences.get(i).getPage());
+                        partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
                    }
                } else {
                    counter = 0;
@@ -99,22 +120,23 @@ public class SearchableText {
                        j--;
                    }
                    crossSequenceParts = new ArrayList<>();
-                    partMatch = new TextPositionSequence(sequences.get(i).getPage());
+                    partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
                }

-                if (j == sequences.get(i).length() - 1 && counter != 0) {
+                if (j == searchSpace.get(i).length() - 1 && counter != 0) {
                    crossSequenceParts.add(partMatch);
                }
            }
        }

        return finalMatches;
+
    }


    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {

-        UUID id = UUID.randomUUID();
+        String id = IdBuilder.buildId(crossSequenceParts);
        List<EntityPositionSequence> result = new ArrayList<>();
        int currentPage = -1;
        EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
@@ -7,9 +7,10 @@ import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;

-import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;

+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
@@ -31,7 +32,21 @@ public class Section {

    private int sectionNumber;

-    private Map<String, String> tabularData;
+    private Map<String, TextBlock> tabularData;
+
+
+    public boolean isVertebrateStudy() {
+        return tabularData != null
+                && tabularData.containsKey("Vertebrate study Y/N")
+                && tabularData.get("Vertebrate study Y/N").getText().equals("Y");
+    }
+
+
+    public boolean isNotVertebrateStudy() {
+        return tabularData != null
+                && tabularData.containsKey("Vertebrate study Y/N")
+                && tabularData.get("Vertebrate study Y/N").getText().equals("N");
+    }


    public boolean contains(String type) {
@@ -163,20 +178,16 @@ public class Section {

    public void highlightCell(String cellHeader, int ruleNumber) {

-        String value = tabularData.get(cellHeader);
+        TextBlock value = tabularData.get(cellHeader);
        if (value == null) {
            log.warn("Could not find any data for {}.", cellHeader);
        } else {
-            Set<Entity> found = findEntities(value, "must_redact");
-            if (CollectionUtils.isEmpty(found)) {
-                log.warn("Could not identify value {} in row.", value);
-            } else {
-                Entity entity = found.iterator().next();
-                entity.setRedaction(false);
-                entity.setMatchedRule(ruleNumber);
-                entity.setRedactionReason(cellHeader);
-                entities.add(entity);
-            }
+            Entity entity = new Entity(value.getText(), "must_redact", 0, value.getText().length(), headline, sectionNumber);
+            entity.setRedaction(false);
+            entity.setMatchedRule(ruleNumber);
+            entity.setRedactionReason(cellHeader);
+            entity.setTargetSequences(value.getSequences()); // Make sure no other cells with same content are highlighted
+            entities.add(entity);
        }

    }
@@ -12,6 +12,9 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;

+import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -23,7 +26,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@Service
@RequiredArgsConstructor
 public class EntityRedactionService {
@@ -32,7 +37,7 @@ public class EntityRedactionService {
    private final DroolsExecutionService droolsExecutionService;


-    public void processDocument(Document classifiedDoc) {
+    public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {

        dictionaryService.updateDictionary();
        droolsExecutionService.updateRules();
@@ -46,23 +51,27 @@ public class EntityRedactionService {
            List<Table> tables = paragraph.getTables();

            for (Table table : tables) {
-                List<String> metadata = table.getHeaders();
                for (List<Cell> row : table.getRows()) {
                    SearchableText searchableRow = new SearchableText();
-                    List<String> cellValues = new ArrayList<>();
+                    Map<String, TextBlock> tabularData = new HashMap<>();
                    for (Cell cell : row) {
-                        if (cell == null || CollectionUtils.isEmpty(cell.getTextBlocks())) {
-                            cellValues.add(null);
+                        if (cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
                            continue;
                        }
-                        cellValues.add(cell.getTextBlocks().get(0).getText());
+                        addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber);
+                        cell.getHeaderCells().forEach(headerCell -> {
+                            String headerName = headerCell.getTextBlocks().get(0).getText()
+                                    .replaceAll("\n", " ")
+                                    .replaceAll(" ", " ");
+                            tabularData.put(headerName, cell.getTextBlocks().get(0));
+                        });
                        for (TextBlock textBlock : cell.getTextBlocks()) {
                            searchableRow.addAll(textBlock.getSequences());
                        }
+
                    }
                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber);

-                    Map<String, String> tabularData = toMap(metadata, cellValues);
                    Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
                            .entities(rowEntities)
                            .text(searchableRow.getAsStringWithLinebreaks())
@@ -78,6 +87,7 @@ public class EntityRedactionService {
                sectionNumber++;
            }

+            addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber);
            Section analysedSection = droolsExecutionService.executeRules(Section.builder()
                    .entities(entities)
@@ -101,8 +111,7 @@ public class EntityRedactionService {
            for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
                classifiedDoc.getEntities()
                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
-                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(),
-                                entity.getRedactionReason(), entry
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber()));
            }
        }
@@ -110,30 +119,15 @@ public class EntityRedactionService {
    }


-    private Map<String, String> toMap(List<String> keys, List<String> values) {
-
-        if (keys.size() != values.size()) {
-            throw new RuntimeException("Cannot merge lists of unequal size.");
-        }
-        Map<String, String> result = new HashMap<>();
-        for (int i = 0; i < keys.size(); i++) {
-            result.put(keys.get(i), values.get(i));
-        }
-
-        return result;
-
-    }
-
-
    private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text) {

        removeEntitiesContainedInLarger(entities);

        for (Entity entity : entities) {
            if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
-                entity.setPositionSequences(text.getSequences(entity.getWord(), true));
+                entity.setPositionSequences(text.getSequences(entity.getWord(), true, entity.getTargetSequences()));
            } else {
-                entity.setPositionSequences(text.getSequences(entity.getWord(), false));
+                entity.setPositionSequences(text.getSequences(entity.getWord(), false, entity.getTargetSequences()));
            }
        }

@@ -177,8 +171,7 @@ public class EntityRedactionService {

                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex,
-                            headline, sectionNumber));
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
                }
            } while (startIndex > -1);
        }
@@ -206,4 +199,23 @@ public class EntityRedactionService {
        entities.removeAll(wordsToRemove);
    }

+
+    private void addSectionToManualRedactions(List<TextBlock> textBlocks, ManualRedactions manualRedactions, String section, int sectionNumber) {
+
+        if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) {
+            return;
+        }
+
+        for (TextBlock textBlock : textBlocks) {
+            for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
+                for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
+                    if (textBlock.contains(rectangle)) {
+                        manualRedactionEntry.setSection(section);
+                        manualRedactionEntry.setSectionNumber(sectionNumber);
+                    }
+                }
+            }
+        }
+    }
+
 }
@@ -0,0 +1,32 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class IdBuilder {
+
+    private final HashFunction hashFunction = Hashing.murmur3_128();
+
+    public String buildId(List<TextPositionSequence> crossSequenceParts) {
+
+        StringBuilder sb = new StringBuilder();
+        crossSequenceParts.forEach(sequencePart -> sequencePart.getTextPositions().forEach(textPosition -> {
+            sb.append(textPosition.getTextMatrix());
+        }));
+
+        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
+    }
+
+
+    public String buildId(ManualRedactionEntry manualRedactionEntry) {
+        return hashFunction.hashString(manualRedactionEntry.toString(), StandardCharsets.UTF_8).toString();
+    }
+}
@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;

 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@@ -13,6 +13,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

@Service
@@ -85,10 +86,20 @@ public class SectionsBuilderService {
                    table.setHeadline("Table in: " + lastHeadline);
                }
                // Distribute header information for subsequent tables
-                if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable) &&
-                        (previousTable.isVerticalHeader() && previousTable.getRowCount() == table.getRowCount() ||
-                                previousTable.getColCount() == table.getColCount())) {
-                    table.setHeaders(previousTable.getHeaders());
+                if (previousTable != null && hasInvalidHeaderInformation(table) && hasValidHeaderInformation(previousTable)) {
+                    List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
+                    List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(table);
+                    if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
+                        for (int i = table.getRows().size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
+                            List<Cell> row = table.getRows().get(i);
+                            if (row.size() == previousTableNonHeaderRow.size()
+                                    && row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
+                                for (int j = 0; j < row.size(); j++) {
+                                    row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
+                                }
+                            }
+                        }
+                    }
                }

                if (textBlock != null && !alreadyAdded) {
@@ -141,14 +152,32 @@ public class SectionsBuilderService {

    private boolean hasInvalidHeaderInformation(Table table) {

-        if (CollectionUtils.isEmpty(table.getHeaders())) {
-            return true;
-        }
-        if (table.getHeaders().stream().anyMatch(StringUtils::isEmpty)) {
-            return true;
+        return table.getRows().stream()
+                .flatMap(row -> row.stream()
+                        .filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
+                .findAny()
+                .isEmpty();
+
+    }
+
+
+    private List<Cell> getRowWithNonHeaderCells(Table table) {
+
+        for (int i = table.getRows().size() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
+            List<Cell> row = table.getRows().get(i);
+            boolean allNonHeader = true;
+            for (Cell cell : row) {
+                if (cell.isHeaderCell()) {
+                    allNonHeader = false;
+                    break;
+                }
+            }
+            if (allNonHeader) {
+                return row;
+            }
        }

-        return false;
+        return Collections.emptyList();

    }

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import com.iqser.red.service.redaction.v1.model.Rectangle;
+
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -22,4 +24,8 @@ public abstract class AbstractTextContainer {
        return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
    }

+    public boolean contains(Rectangle other) {
+        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY()  && this.maxY >= other.getTopLeft().getY() + other.getHeight();
+    }
+
 }
@@ -16,10 +16,14 @@ public class Cell extends Rectangle {

    private List<TextBlock> textBlocks = new ArrayList<>();

+    private List<Cell> headerCells = new ArrayList<>();
+
+    private boolean isHeaderCell;

    public Cell(Point2D topLeft, Point2D bottomRight) {

-        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight
+        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()),
+                (float) (bottomRight
                .getY() - topLeft.getY()));
    }

@@ -29,4 +33,4 @@ public class Cell extends Rectangle {
        textBlocks.add(textBlock);
    }

-}
+}
@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.server.tableextraction.model;
+
+import lombok.RequiredArgsConstructor;
+import lombok.Value;
+
+@Value
+@RequiredArgsConstructor
+public class CellPosition implements Comparable<CellPosition> {
+
+    int row;
+
+    int col;
+
+
+    @Override
+    public int compareTo(CellPosition other) {
+
+        int rowDiff = row - other.row;
+        return rowDiff != 0 ? rowDiff : col - other.col;
+    }
+
+}
@@ -8,12 +8,10 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;

 import lombok.Getter;
@@ -41,12 +39,6 @@ public class Table extends AbstractTextContainer {

    private List<List<Cell>> rows;

-    @Getter
-    @Setter
-    private List<String> headers;
-
-    @Getter
-    private boolean verticalHeader;

    public Table(List<Cell> cells, Rectangle area, int rotation) {

@@ -65,7 +57,7 @@ public class Table extends AbstractTextContainer {

        if (rows == null) {
            rows = computeRows();
-            headers = computeHeaders();
+            computeHeaders();
        }

        return rows;
@@ -78,69 +70,105 @@ public class Table extends AbstractTextContainer {
     * Column is marked as header if cell text is bold and row cell text is not bold.
     * Defaults to row.
     */
-    private List<String> computeHeaders() {
+    private void computeHeaders() {

-        boolean allBold = true;
-        List<Cell> rowCells = rows.get(0);
-        for (Cell cell : rowCells) {
-            if (cell == null || CollectionUtils.isEmpty(cell.getTextBlocks()) ||
-                    !cell.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
-                allBold = false;
-                break;
-            }
-        }
-        if (!allBold) {
-            allBold = true;
-            List<Cell> firstColCells = new ArrayList<>();
-            for (List<Cell> row : rows) {
-                Cell firstInRow = row.get(0);
-                if (firstInRow == null || CollectionUtils.isEmpty(firstInRow.getTextBlocks()) ||
-                        !firstInRow.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
-                    allBold = false;
+        // A bold cell is a header cell as long as every cell to the left/top is bold, too
+        cells.forEach((position, cell) -> {
+            List<Cell> cellsToTheLeft = getCellsToTheLeft(position);
+            Cell lastHeaderCell = null;
+            for (Cell leftCell : cellsToTheLeft) {
+                if (CollectionUtils.isNotEmpty(leftCell.getTextBlocks()) && leftCell.getTextBlocks()
+                        .get(0)
+                        .getMostPopularWordStyle()
+                        .equals("bold")) {
+                    lastHeaderCell = leftCell;
+                } else {
                    break;
                }
-                firstColCells.add(firstInRow);
            }
-            if (allBold) {
-                log.info("Headers are in first column");
-                verticalHeader = true;
-                return firstColCells.stream().map(cell -> {
-                    if (CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
-                        return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
-                                .replaceAll("\n", " ")
-                                .replaceAll(" ", " ");
-                    } else {
-                        return null;
-                    }
-                }).collect(Collectors.toList());
-            } else {
-                log.info("Headers are defaulted in first row.");
-                return rowCells.stream().map(cell -> {
-                    if (cell != null && CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
-                        return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
-                                .replaceAll("\n", " ")
-                                .replaceAll(" ", " ");
-                    } else {
-                        return null;
-                    }
-                }).collect(Collectors.toList());
+            if (lastHeaderCell != null) {
+                cell.getHeaderCells().add(lastHeaderCell);
            }
-        } else {
-            log.info("Headers are in first row.");
-            return rowCells.stream().map(cell -> {
-                if (CollectionUtils.isNotEmpty(cell.getTextBlocks())) {
-                    return TextNormalizationUtilities.removeHyphenLineBreaks(cell.getTextBlocks().get(0).getText())
-                            .replaceAll("\n", " ")
-                            .replaceAll(" ", " ");
+            lastHeaderCell = null;
+            List<Cell> cellsToTheTop = getCellToTheTop(position);
+            for (Cell topCell : cellsToTheTop) {
+                if (CollectionUtils.isNotEmpty(topCell.getTextBlocks()) && topCell.getTextBlocks()
+                        .get(0)
+                        .getMostPopularWordStyle()
+                        .equals("bold")) {
+                    lastHeaderCell = topCell;
                } else {
-                    return null;
+                    break;
                }
-            }).collect(Collectors.toList());
-        }
+            }
+            if (lastHeaderCell != null) {
+                cell.getHeaderCells().add(lastHeaderCell);
+            }
+            if (CollectionUtils.isNotEmpty(cell.getTextBlocks()) && cell.getTextBlocks()
+                    .get(0)
+                    .getMostPopularWordStyle()
+                    .equals("bold")) {
+                cell.setHeaderCell(true);
+            }
+        });

    }


+    private List<Cell> getCellsToTheLeft(CellPosition cellPosition) {
+
+        List<Cell> result = new ArrayList<>();
+        if (cellPosition.getCol() == 0) {
+            return result;
+        }
+        int row = cellPosition.getRow();
+        for (int i = cellPosition.getCol() - 1; i >= 0; i--) {
+            if (cells.get(new CellPosition(row, i)) != null) {
+                result.add(cells.get(new CellPosition(row, i)));
+            } else {
+                Cell spanningCell = null;
+                while (spanningCell == null && row >= 0) {
+                    row--;
+                    spanningCell = cells.get(new CellPosition(row, i));
+                }
+                if (spanningCell != null) {
+                    result.add(spanningCell);
+                }
+                row = cellPosition.getRow();
+            }
+        }
+        Collections.reverse(result);
+        return result;
+    }
+
+
+    private List<Cell> getCellToTheTop(CellPosition cellPosition) {
+
+        List<Cell> result = new ArrayList<>();
+        if (cellPosition.getRow() == 0) {
+            return result;
+        }
+        int col = cellPosition.getCol();
+        for (int i = cellPosition.getRow() - 1; i >= 0; i--) {
+            if (cells.get(new CellPosition(i, col)) != null) {
+                result.add(cells.get(new CellPosition(i, col)));
+            } else {
+                Cell spanningCell = null;
+                while (spanningCell == null && col >= 0) {
+                    col--;
+                    spanningCell = cells.get(new CellPosition(i, col));
+                }
+                if (spanningCell != null) {
+                    result.add(spanningCell);
+                }
+                col = cellPosition.getCol();
+            }
+        }
+        Collections.reverse(result);
+        return result;
+    }
+
+
    private List<List<Cell>> computeRows() {

        List<List<Cell>> rows = new ArrayList<>();
@@ -149,7 +177,9 @@ public class Table extends AbstractTextContainer {
                List<Cell> lastRow = new ArrayList<>();
                for (int j = rowCount - 1; j >= 0; j--) { // cols
                    Cell cell = cells.get(new CellPosition(j, i));
-                    lastRow.add(cell);
+                    if (cell != null) {
+                        lastRow.add(cell);
+                    }
                }
                rows.add(lastRow);
            }
@@ -158,7 +188,9 @@ public class Table extends AbstractTextContainer {
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < rowCount; j++) { // cols
                    Cell cell = cells.get(new CellPosition(i, j));
-                    lastRow.add(cell);
+                    if (cell != null) {
+                        lastRow.add(cell);
+                    }
                }
                rows.add(lastRow);
            }
@@ -167,7 +199,9 @@ public class Table extends AbstractTextContainer {
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < colCount; j++) {
                    Cell cell = cells.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
-                    lastRow.add(cell);
+                    if (cell != null) {
+                        lastRow.add(cell);
+                    }
                }
                rows.add(lastRow);
            }
@@ -217,20 +251,21 @@ public class Table extends AbstractTextContainer {
            while (rowCells.hasNext()) {
                Cell cell = rowCells.next();
                if (i > 0) {
-                    List<List<Cell>> others = rowsOfCells(si.contains(new Rectangle(cell.getBottom(), si.getBounds()
-                            .getLeft(), cell.getLeft() - si.getBounds().getLeft() + 1, si.getBounds().getBottom() - cell
-                            .getBottom())));
+                    Rectangle rectangle = new Rectangle(cell.getBottom(),
+                            si.getBounds().getLeft(),
+                            cell.getLeft() - si.getBounds().getLeft() + 1,
+                            si.getBounds().getBottom() - cell.getBottom());
+                    List<List<Cell>> others = rowsOfCells(si.contains(rectangle));

                    for (List<Cell> r : others) {
                        jumpToColumn = Math.max(jumpToColumn, r.size());
                    }
-                }

-                while (startColumn != jumpToColumn) {
-                    add(previousNonNullCellForColumnIndex.get(startColumn), i, startColumn);
-                    startColumn++;
+                    while (startColumn != jumpToColumn) {
+                        add(previousNonNullCellForColumnIndex.get(startColumn), i, startColumn);
+                        startColumn++;
+                    }
                }
-
                add(cell, i, startColumn);
                previousNonNullCellForColumnIndex.put(startColumn, cell);
                startColumn++;
@@ -240,26 +275,23 @@ public class Table extends AbstractTextContainer {
    }


-    private static List<List<Cell>> rowsOfCells(List<Cell> cells) {
+    private List<List<Cell>> rowsOfCells(List<Cell> cells) {

-        Cell c;
-        float lastTop;
        List<List<Cell>> rv = new ArrayList<>();
-        List<Cell> lastRow;

        if (cells.isEmpty()) {
            return rv;
        }
-
        cells.sort(Comparator.comparingDouble(Rectangle::getLeft));

-        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1
+        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2),
+                Utils.round(arg1
                .getBottom(), 2))));

        Iterator<Cell> iter = cells.iterator();
-        c = iter.next();
-        lastTop = c.getBottom();
-        lastRow = new ArrayList<>();
+        Cell c = iter.next();
+        float lastTop = c.getBottom();
+        List<Cell> lastRow = new ArrayList<>();
        lastRow.add(c);
        rv.add(lastRow);

@@ -346,51 +378,4 @@ public class Table extends AbstractTextContainer {
        return sb.toString();
    }

-
-    static class CellPosition implements Comparable<CellPosition> {
-
-        CellPosition(int row, int col) {
-
-            this.row = row;
-            this.col = col;
-        }
-
-
-        final int row;
-        final int col;
-
-
-        @Override
-        public int hashCode() {
-
-            return row + 101 * col;
-        }
-
-
-        @Override
-        public boolean equals(Object obj) {
-
-            if (this == obj) {
-                return true;
-            }
-            if (obj == null) {
-                return false;
-            }
-            if (getClass() != obj.getClass()) {
-                return false;
-            }
-            CellPosition other = (CellPosition) obj;
-            return row == other.row && col == other.col;
-        }
-
-
-        @Override
-        public int compareTo(CellPosition other) {
-
-            int rowDiff = row - other.row;
-            return rowDiff != 0 ? rowDiff : col - other.col;
-        }
-
-    }
-
 }
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.service;

 import java.awt.geom.Point2D;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -25,26 +24,28 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;

@Service
-@SuppressWarnings("all")
 public class TableExtractionService {

-    public void extractTables(CleanRulings cleanRulings, Page page){
+    public void extractTables(CleanRulings cleanRulings, Page page) {

        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

-        Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
-        while (itty.hasNext()) {
-            TextBlock textBlock = (TextBlock) itty.next();
+        for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
+            TextBlock textBlock = (TextBlock) abstractTextContainer;
            for (Cell cell : cells) {
-                if (cell.intersects(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight())) {
+                if (cell.intersects(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(),
+                        textBlock.getHeight())) {
                    cell.addTextBlock(textBlock);
                    break;
                }
            }
        }

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells)
-                .stream()
+        cells = new ArrayList<>(new HashSet<>(cells));
+        Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
+
+
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream()
                .filter(r -> r.getWidth() > 0f && r.getHeight() > 0f)
                .collect(Collectors.toList());

@@ -63,9 +64,9 @@ public class TableExtractionService {
        for (Table table : tables) {
            int position = -1;

-            itty = page.getTextBlocks().iterator();
+            Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
            while (itty.hasNext()) {
-                AbstractTextContainer textBlock = (AbstractTextContainer) itty.next();
+                AbstractTextContainer textBlock = itty.next();
                if (table.contains(textBlock)) {
                    if (position == -1) {
                        position = page.getTextBlocks().indexOf(textBlock);
@@ -79,17 +80,18 @@ public class TableExtractionService {
        }
    }

+
    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+
        List<Cell> cellsFound = new ArrayList<>();
-        Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
+        Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines,
+                verticalRulingLines);
        List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
-        Collections.sort(intersectionPointsList, POINT_COMPARATOR);
-        boolean doBreak;
+        intersectionPointsList.sort(POINT_COMPARATOR);

        for (int i = 0; i < intersectionPointsList.size(); i++) {
            Point2D topLeft = intersectionPointsList.get(i);
            Ruling[] hv = intersectionPoints.get(topLeft);
-            doBreak = false;

            // CrossingPointsDirectlyBelow( topLeft );
            List<Point2D> xPoints = new ArrayList<>();
@@ -106,10 +108,6 @@ public class TableExtractionService {
            }
            outer:
            for (Point2D xPoint : xPoints) {
-                if (doBreak) {
-                    break;
-                }
-
                // is there a vertical edge b/w topLeft and xPoint?
                if (!hv[1].equals(intersectionPoints.get(xPoint)[1])) {
                    continue;
@@ -120,11 +118,9 @@ public class TableExtractionService {
                        continue;
                    }
                    Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
-                    if (intersectionPoints.containsKey(btmRight)
-                            && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0])
-                            && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
+                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints
+                            .get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
                        cellsFound.add(new Cell(topLeft, btmRight));
-                        doBreak = true;
                        break outer;
                    }
                }
@@ -139,7 +135,7 @@ public class TableExtractionService {
    }


-    public List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
+    private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
        // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
        List<Rectangle> rectangles = new ArrayList<>();
        Set<Point2D> pointSet = new HashSet<>();
@@ -147,10 +143,6 @@ public class TableExtractionService {
        Map<Point2D, Point2D> edgesV = new HashMap<>();
        int i = 0;

-        cells = new ArrayList<>(new HashSet<>(cells));
-
-        Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
-
        for (Rectangle cell : cells) {
            for (Point2D pt : cell.getPoints()) {
                if (pointSet.contains(pt)) { // shared vertex, remove it
@@ -163,10 +155,10 @@ public class TableExtractionService {

        // X first sort
        List<Point2D> pointsSortX = new ArrayList<>(pointSet);
-        Collections.sort(pointsSortX, X_FIRST_POINT_COMPARATOR);
+        pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
        // Y first sort
        List<Point2D> pointsSortY = new ArrayList<>(pointSet);
-        Collections.sort(pointsSortY, POINT_COMPARATOR);
+        pointsSortY.sort(POINT_COMPARATOR);

        while (i < pointSet.size()) {
            float currY = (float) pointsSortY.get(i).getY();
@@ -203,13 +195,12 @@ public class TableExtractionService {
                    nextVertex = edgesV.get(curr.point);
                    edgesV.remove(curr.point);
                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.VERTICAL);
-                    polygon.add(lastAddedVertex);
                } else {
                    nextVertex = edgesH.get(curr.point);
                    edgesH.remove(curr.point);
                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.HORIZONTAL);
-                    polygon.add(lastAddedVertex);
                }
+                polygon.add(lastAddedVertex);

                if (lastAddedVertex.equals(polygon.get(0))) {
                    // closed polygon
@@ -227,10 +218,10 @@ public class TableExtractionService {

        // calculate grid-aligned minimum area rectangles for each found polygon
        for (List<PolygonVertex> poly : polygons) {
-            float top = java.lang.Float.MAX_VALUE;
-            float left = java.lang.Float.MAX_VALUE;
-            float bottom = java.lang.Float.MIN_VALUE;
-            float right = java.lang.Float.MIN_VALUE;
+            float top = Float.MAX_VALUE;
+            float left = Float.MAX_VALUE;
+            float bottom = Float.MIN_VALUE;
+            float right = Float.MIN_VALUE;
            for (PolygonVertex pt : poly) {
                top = (float) Math.min(top, pt.point.getY());
                left = (float) Math.min(left, pt.point.getX());
@@ -244,69 +235,66 @@ public class TableExtractionService {
    }


-    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = new Comparator<Point2D>() {
-        @Override
-        public int compare(Point2D arg0, Point2D arg1) {
-            int rv = 0;
-            float arg0X = Utils.round(arg0.getX(), 2);
-            float arg0Y = Utils.round(arg0.getY(), 2);
-            float arg1X = Utils.round(arg1.getX(), 2);
-            float arg1Y = Utils.round(arg1.getY(), 2);
+    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {

-            if (arg0X > arg1X) {
-                rv = 1;
-            } else if (arg0X < arg1X) {
-                rv = -1;
-            } else if (arg0Y > arg1Y) {
-                rv = 1;
-            } else if (arg0Y < arg1Y) {
-                rv = -1;
-            }
-            return rv;
+        int rv = 0;
+        float arg0X = Utils.round(arg0.getX(), 2);
+        float arg0Y = Utils.round(arg0.getY(), 2);
+        float arg1X = Utils.round(arg1.getX(), 2);
+        float arg1Y = Utils.round(arg1.getY(), 2);
+
+        if (arg0X > arg1X) {
+            rv = 1;
+        } else if (arg0X < arg1X) {
+            rv = -1;
+        } else if (arg0Y > arg1Y) {
+            rv = 1;
+        } else if (arg0Y < arg1Y) {
+            rv = -1;
        }
+        return rv;
    };

+    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {

-    private static final Comparator<Point2D> POINT_COMPARATOR = new Comparator<Point2D>() {
-        @Override
-        public int compare(Point2D arg0, Point2D arg1) {
-            int rv = 0;
-            float arg0X = Utils.round(arg0.getX(), 2);
-            float arg0Y = Utils.round(arg0.getY(), 2);
-            float arg1X = Utils.round(arg1.getX(), 2);
-            float arg1Y = Utils.round(arg1.getY(), 2);
+        int rv = 0;
+        float arg0X = Utils.round(arg0.getX(), 2);
+        float arg0Y = Utils.round(arg0.getY(), 2);
+        float arg1X = Utils.round(arg1.getX(), 2);
+        float arg1Y = Utils.round(arg1.getY(), 2);

-
-            if (arg0Y > arg1Y) {
-                rv = 1;
-            } else if (arg0Y < arg1Y) {
-                rv = -1;
-            } else if (arg0X > arg1X) {
-                rv = 1;
-            } else if (arg0X < arg1X) {
-                rv = -1;
-            }
-            return rv;
+        if (arg0Y > arg1Y) {
+            rv = 1;
+        } else if (arg0Y < arg1Y) {
+            rv = -1;
+        } else if (arg0X > arg1X) {
+            rv = 1;
+        } else if (arg0X < arg1X) {
+            rv = -1;
        }
+        return rv;
    };

-
    private enum Direction {
-        HORIZONTAL,
-        VERTICAL
+        HORIZONTAL, VERTICAL
    }

    static class PolygonVertex {
+
        Point2D point;
        Direction direction;

-        public PolygonVertex(Point2D point, Direction direction) {
+
+        PolygonVertex(Point2D point, Direction direction) {
+
            this.direction = direction;
            this.point = point;
        }

+
        @Override
        public boolean equals(Object other) {
+
            if (this == other) {
                return true;
            }
@@ -316,15 +304,21 @@ public class TableExtractionService {
            return this.point.equals(((PolygonVertex) other).point);
        }

+
        @Override
        public int hashCode() {
+
            return this.point.hashCode();
        }

+
        @Override
        public String toString() {
-            return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
+
+            return String.format("%s[point=%s,direction=%s]", this.getClass()
+                    .getName(), this.point.toString(), this.direction.toString());
        }
+
    }

 }
@@ -16,7 +16,8 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -41,138 +43,171 @@ public class AnnotationHighlightService {
    private final DictionaryService dictionaryService;


-    public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {
+    public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction, ManualRedactions manualRedactions) throws IOException {

        for (int page = 1; page <= document.getNumberOfPages(); page++) {

            PDPage pdPage = document.getPage(page - 1);

-            if (!flatRedaction) {
-                PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-                for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
-
-                    for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
-
-                        AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
-
-                        if (textBlock.getPage() != page) {
-                            continue;
-                        }
-                        if (textBlock instanceof TextBlock) {
-                            textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                            visualizeTextBlock((TextBlock) textBlock, contentStream);
-                        } else if (textBlock instanceof Table) {
-                            textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                            visualizeTable((Table) textBlock, contentStream);
-                        }
-
-                    }
-                }
-
-                contentStream.close();
-            }
+            drawSectionFrames(document, classifiedDoc, flatRedaction, pdPage, page);

            if (classifiedDoc.getEntities().get(page) == null) {
                continue;
            }

-            for (Entity entity : classifiedDoc.getEntities().get(page)) {
+            addAnnotations(pdPage, classifiedDoc, flatRedaction, manualRedactions, page);
+            addManualAnnotations(pdPage, classifiedDoc, manualRedactions, page);
+        }
+    }

-                RedactionLogEntry redactionLogEntry = new RedactionLogEntry();

-                for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+    private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction, ManualRedactions manualRedactions, int page) throws IOException {

-                    if (flatRedaction && !isRedactionType(entity)) {
-                        continue;
-                    }
+        List<PDAnnotation> annotations = pdPage.getAnnotations();

-                    for (TextPositionSequence textPositions : entityPositionSequence.getSequences()) {
+        for (Entity entity : classifiedDoc.getEntities().get(page)) {

-                        float height = textPositions.getTextPositions().get(0).getHeightDir() + 2;
-
-                        float posXInit;
-                        float posXEnd;
-                        float posYInit;
-                        float posYEnd;
-
-                        if (textPositions.getTextPositions().get(0).getRotation() == 90) {
-
-                            posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
-                            posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
-                            posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posYEnd = textPositions.getTextPositions()
-                                    .get(textPositions.getTextPositions().size() - 1)
-                                    .getXDirAdj() - height + 4;
-                        } else {
-
-                            posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posXEnd = textPositions.getTextPositions()
-                                    .get(textPositions.getTextPositions().size() - 1)
-                                    .getXDirAdj() + textPositions.getTextPositions()
-                                    .get(textPositions.getTextPositions().size() - 1)
-                                    .getWidth() + 1;
-                            posYInit = textPositions.getTextPositions()
-                                    .get(0)
-                                    .getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
-                            posYEnd = textPositions.getTextPositions()
-                                    .get(0)
-                                    .getPageHeight() - textPositions.getTextPositions()
-                                    .get(textPositions.getTextPositions().size() - 1)
-                                    .getYDirAdj() + 2;
-                        }
-
-                        Rectangle textHighlightRectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
-
-                        List<PDAnnotation> annotations = pdPage.getAnnotations();
-                        PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
-                        highlight.constructAppearances();
-
-                        PDRectangle annotationPosition = new PDRectangle();
-                        annotationPosition.setLowerLeftX(posXInit);
-                        annotationPosition.setLowerLeftY(posYEnd);
-                        annotationPosition.setUpperRightX(posXEnd);
-                        annotationPosition.setUpperRightY(posYEnd + height);
-
-                        highlight.setRectangle(annotationPosition);
-                        if (!flatRedaction && !isHint(entity)) {
-                            highlight.setAnnotationName(entityPositionSequence.getId().toString());
-                            highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity
-                                    .getHeadline() + "\"");
-                        }
-
-                        highlight.setQuadPoints(toQuadPoints(textHighlightRectangle));
-
-                        PDColor color;
-                        if (flatRedaction) {
-                            color = new PDColor(new float[]{0, 0, 0}, PDDeviceRGB.INSTANCE);
-                        } else {
-                            color = new PDColor(getColor(entity), PDDeviceRGB.INSTANCE);
-                        }
-
-                        highlight.setColor(color);
-                        annotations.add(highlight);
-
-                        redactionLogEntry.getPositions().add(textHighlightRectangle);
-
-                    }
-                    redactionLogEntry.setId(entityPositionSequence.getId().toString());
-                }
-                redactionLogEntry.setColor(getColor(entity));
-                redactionLogEntry.setReason(entity.getRedactionReason());
-                redactionLogEntry.setValue(entity.getWord());
-                redactionLogEntry.setType(entity.getType());
-                redactionLogEntry.setRedacted(entity.isRedaction());
-                redactionLogEntry.setSection(entity.getHeadline());
-                redactionLogEntry.setHint(isHint(entity));
-                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
+            if (flatRedaction && !isRedactionType(entity)) {
+                continue;
            }

+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity);
+
+            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+
+                if (manualRedactions != null && manualRedactions.getIdsToRemove()
+                        .contains(entityPositionSequence.getId())) {
+                    entity.setRedaction(false);
+                    entity.setRedactionReason(entity.getRedactionReason() + ", removed by manual override");
+                    redactionLogEntry.setManual(true);
+                }
+
+                for (TextPositionSequence textPositions : entityPositionSequence.getSequences()) {
+
+                    Rectangle rectangle = textPositions.getRectangle();
+                    redactionLogEntry.getPositions().add(rectangle);
+                    annotations.add(createAnnotation(rectangle, entityPositionSequence.getId(), createAnnotationContent(entity), getColor(entity), !flatRedaction && !isHint(entity)));
+                }
+                redactionLogEntry.setId(entityPositionSequence.getId());
+            }
+            classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
        }
    }


+    private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions, int page) throws IOException {
+
+        if (manualRedactions == null) {
+            return;
+        }
+
+        List<PDAnnotation> annotations = pdPage.getAnnotations();
+
+        for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
+
+            String id = IdBuilder.buildId(manualRedactionEntry);
+
+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry);
+
+            for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
+
+                if (page != rectangle.getPage()) {
+                    continue;
+                }
+
+                PDAnnotationTextMarkup highlight = createAnnotation(rectangle, id, createAnnotationContent(manualRedactionEntry), getColor(manualRedactionEntry
+                        .getType()), true);
+                annotations.add(highlight);
+
+                redactionLogEntry.getPositions().add(rectangle);
+            }
+            classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
+        }
+    }
+
+
+    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry) {
+
+        return RedactionLogEntry.builder()
+                .color(getColor(manualRedactionEntry.getType()))
+                .reason(manualRedactionEntry.getReason())
+                .value(manualRedactionEntry.getValue())
+                .type(manualRedactionEntry.getType())
+                .redacted(true)
+                .isHint(false)
+                .section(manualRedactionEntry.getSection())
+                .sectionNumber(manualRedactionEntry.getSectionNumber())
+                .manual(true)
+                .build();
+    }
+
+
+    private RedactionLogEntry createRedactionLogEntry(Entity entity) {
+
+        return RedactionLogEntry.builder()
+                .color(getColor(entity))
+                .reason(entity.getRedactionReason())
+                .value(entity.getWord())
+                .type(entity.getType())
+                .redacted(entity.isRedaction())
+                .isHint(isHint(entity))
+                .section(entity.getHeadline())
+                .sectionNumber(entity.getSectionNumber())
+                .build();
+    }
+
+
+    private PDAnnotationTextMarkup createAnnotation(Rectangle rectangle, String id, String content, float[] color, boolean popup) {
+
+        PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
+        annotation.constructAppearances();
+        annotation.setRectangle(toPDRectangle(rectangle));
+        annotation.setQuadPoints(toQuadPoints(rectangle));
+        if (popup) {
+            annotation.setAnnotationName(id);
+            annotation.setTitlePopup(id);
+            annotation.setContents(content);
+        }
+        annotation.setColor(new PDColor(color, PDDeviceRGB.INSTANCE));
+        return annotation;
+    }
+
+
+    private String createAnnotationContent(Entity entity) {
+
+        return new StringBuilder().append("\nRule ")
+                .append(entity.getMatchedRule())
+                .append(" matched")
+                .append("\n\n")
+                .append(entity.getRedactionReason())
+                .append("\n\nIn Section : \"")
+                .append(entity.getHeadline())
+                .append("\"")
+                .toString();
+    }
+
+
+    private String createAnnotationContent(ManualRedactionEntry entry) {
+
+        return new StringBuilder().append("\nManual Redaction")
+                .append("\n\nIn Section : \"")
+                .append(entry.getSection())
+                .append("\"")
+                .toString();
+    }
+
+
+    private PDRectangle toPDRectangle(Rectangle rectangle) {
+
+        PDRectangle annotationPosition = new PDRectangle();
+        annotationPosition.setLowerLeftX(rectangle.getTopLeft().getX());
+        annotationPosition.setLowerLeftY(rectangle.getTopLeft().getY() + rectangle.getHeight());
+        annotationPosition.setUpperRightX(rectangle.getTopLeft().getX() + rectangle.getWidth());
+        annotationPosition.setUpperRightY(rectangle.getTopLeft().getY());
+        return annotationPosition;
+    }
+
+
    private float[] toQuadPoints(Rectangle rectangle) {

        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
@@ -201,15 +236,24 @@ public class AnnotationHighlightService {
        if (!entity.isRedaction() && !isHint(entity)) {
            return new float[]{0.627f, 0.627f, 0.627f};
        }
-
        if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
            return dictionaryService.getDefaultColor();
        }
-
        return dictionaryService.getEntryColors().get(entity.getType());
    }

+
+    private float[] getColor(String type) {
+
+        if (!dictionaryService.getEntryColors().containsKey(type)) {
+            return dictionaryService.getDefaultColor();
+        }
+        return dictionaryService.getEntryColors().get(type);
+    }
+
+
    private boolean isHint(Entity entity) {
+
        List<String> hintTypes = dictionaryService.getHintTypes();
        if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
            return true;
@@ -217,24 +261,49 @@ public class AnnotationHighlightService {
        return false;
    }

+
+    private void drawSectionFrames(PDDocument document, Document classifiedDoc, boolean flatRedaction, PDPage pdPage, int page) throws IOException {
+
+        if (flatRedaction) {
+            return;
+        }
+
+        PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+
+            for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
+
+                AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
+
+                if (textBlock.getPage() != page) {
+                    continue;
+                }
+                if (textBlock instanceof TextBlock) {
+                    textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
+                    visualizeTextBlock((TextBlock) textBlock, contentStream);
+                } else if (textBlock instanceof Table) {
+                    textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
+                    visualizeTable((Table) textBlock, contentStream);
+                }
+            }
+        }
+        contentStream.close();
+    }
+
+
    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {

        contentStream.setStrokingColor(Color.LIGHT_GRAY);
        contentStream.setLineWidth(0.5f);
-
        contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
        contentStream.stroke();

        if (textBlock.getClassification() != null) {
            contentStream.beginText();
-
            contentStream.setNonStrokingColor(Color.DARK_GRAY);
            contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
-
            contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
-
            contentStream.showText(textBlock.getClassification());
-
            contentStream.endText();
        }
    }
@@ -251,26 +320,16 @@ public class AnnotationHighlightService {
                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
                            .getHeight());
                    contentStream.stroke();
-
-//                    contentStream.setStrokingColor(Color.GREEN);
-//                    for (TextBlock textBlock : cell.getTextBlocks()) {
-//                        contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
-//                        contentStream.stroke();
-//                    }
                }
            }
        }

        if (table.getClassification() != null) {
            contentStream.beginText();
-
            contentStream.setNonStrokingColor(Color.DARK_GRAY);
            contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
-
            contentStream.newLineAtOffset(table.getMinX(), table.getMinY());
-
            contentStream.showText(table.getClassification());
-
            contentStream.endText();
        }
    }
@@ -5,6 +5,8 @@ import static org.springframework.boot.test.context.SpringBootTest.WebEnvironmen

 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -15,11 +17,11 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -40,6 +42,10 @@ import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -48,7 +54,6 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

-@Ignore
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {
@@ -219,11 +224,52 @@ public class RedactionIntegrationTest {
    }


+    @Test
+    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
+
+        System.out.println("noExceptionShouldBeThrownForAnyFiles");
+        ClassLoader loader = getClass().getClassLoader();
+        URL url = loader.getResource("files");
+        File[] files = new File(url.getPath()).listFiles();
+        List<File> input = new ArrayList<>();
+        for (File file : files) {
+            input.addAll(getPathsRecursively(file));
+        }
+        for (File path : input) {
+            RedactionRequest request = RedactionRequest.builder()
+                    .document(IOUtils.toByteArray(new FileInputStream(path)))
+                    .build();
+            System.out.println("Redacting file : " + path.getName());
+            redactionController.redact(request);
+        }
+
+    }
+
+
+    private List<File> getPathsRecursively(File path) {
+
+        List<File> result = new ArrayList<>();
+        if (path == null || path.listFiles() == null) {
+            return result;
+        }
+        for (File f : path.listFiles()) {
+            if (f.isFile()) {
+                result.add(f);
+            } else {
+                result.addAll(getPathsRecursively(f));
+            }
+        }
+        return result;
+
+    }
+
+
    @Test
    public void redactionTest() throws IOException {

+        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Trinexapac/96 Trinexapac-ethyl_RAR_09_Volume_3CA_B-7_2018-02-23.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -241,11 +287,13 @@ public class RedactionIntegrationTest {
        System.out.println("numberOfPages: " + result.getNumberOfPages());
    }

+
    @Test
    public void testTableRedaction() throws IOException {

+        System.out.println("testTableRedaction");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -263,10 +311,47 @@ public class RedactionIntegrationTest {
    }


+    @Test
+    public void testManualRedaction() throws IOException {
+
+        System.out.println("testManualRedaction");
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.setIdsToRemove(Set.of("0836727c3508a0b2ea271da69c04cc2f"));
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setType("name");
+        manualRedactionEntry.setValue("O'Loughlin C.K.");
+        manualRedactionEntry.setReason("Manual Redaction");
+        manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1)));
+
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
+
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .manualRedactions(manualRedactions)
+                .build();
+
+        RedactionResult result = redactionController.redact(request);
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
+            fileOutputStream.write(result.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
    @Test
    public void classificationTest() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
+        System.out.println("classificationTest");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " +
+                "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -283,7 +368,9 @@ public class RedactionIntegrationTest {
    @Test
    public void sectionsTest() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
+        System.out.println("sectionsTest");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " +
+                "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -300,7 +387,9 @@ public class RedactionIntegrationTest {
    @Test
    public void htmlTablesTest() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
+        System.out.println("htmlTablesTest");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " +
+                "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -317,7 +406,9 @@ public class RedactionIntegrationTest {
    @Test
    public void htmlTableRotationTest() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        System.out.println("htmlTableRotationTest");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S" +
+                "-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -337,7 +428,8 @@ public class RedactionIntegrationTest {
        if (resource == null) {
            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(),
+                StandardCharsets.UTF_8))) {
            StringBuilder sb = new StringBuilder();
            String str;
            while ((str = br.readLine()) != null) {
@@ -10,13 +10,16 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;

 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -45,14 +48,15 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;

-@RunWith(SpringRunner.class)
@SpringBootTest
+@RunWith(SpringRunner.class)
 public class EntityRedactionServiceTest {

    private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
    private static final String NAME_CODE = "name";
    private static final String ADDRESS_CODE = "address";

+    private static final AtomicLong DICTIONARY_VERSION = new AtomicLong();
    @MockBean
    private DictionaryClient dictionaryClient;

@@ -112,6 +116,111 @@ public class EntityRedactionServiceTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

+        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
+                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
+                .build();
+        when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+
+        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc, null);
+            assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
+            assertThat(classifiedDoc.getEntities().get(1)).hasSize(5); // 4 out of 5 entities recognized on page 1
+        }
+    }
+
+
+    @Test
+    public void testTrueNegativesInTable() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
+                " Supplement - Identity of the active substance - Reference list.pdf");
+        when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
+                .build();
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc, null);
+            assertThat(classifiedDoc.getEntities()
+                    .entrySet()
+                    .stream()
+                    .noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue();
+        }
+        pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " +
+                "the plant protection product.pdf");
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc, null);
+            assertThat(classifiedDoc.getEntities()
+                    .entrySet()
+                    .stream()
+                    .noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue();
+        }
+    }
+
+    @Test
+    public void testFalsePositiveInWrongCell() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
+        when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
+                .build();
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc, null);
+            assertThat(classifiedDoc.getEntities()).hasSize(1); // two pages
+            assertThat(classifiedDoc.getEntities().get(1).stream()
+                    .filter(entity -> entity.getMatchedRule() == 9)
+                    .count()).isEqualTo(10);
+        }
+
+    }
+
+    @Test
+    public void headerPropagation() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation.pdf");
+
+        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
+                .entries(Arrays.asList("Bissig R.", "Thanei P."))
+                .build();
+
+        when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
+            entityRedactionService.processDocument(classifiedDoc, null);
+            assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
+            assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
+            assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(4);
+        }
+    }
+
+
+    @Before
+    public void stubRedaction() {
        String tableRules = "package drools\n" +
                "\n" +
                "import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
@@ -119,10 +228,7 @@ public class EntityRedactionServiceTest {
                "global Section section\n" +
                "rule \"9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" +
                "    when\n" +
-                "        Section(tabularData != null && tabularData.size() > 0\n" +
-                "            && tabularData.containsKey(\"Vertebrate study Y/N\")\n" +
-                "            && tabularData.get(\"Vertebrate study Y/N\").equals(\"Y\")\n" +
-                "        )\n" +
+                "        Section(isVertebrateStudy())\n" +
                "    then\n" +
                "        section.redact(\"name\", 9, \"Redacted because row is a vertebrate study\");\n" +
                "        section.redact(\"address\", 9, \"Redacted because rows is a vertebrate study\");\n" +
@@ -135,22 +241,9 @@ public class EntityRedactionServiceTest {
                        TypeResult.builder().type(NAME_CODE).color(new float[]{1, 1, 0}).build(),
                        TypeResult.builder().type(ADDRESS_CODE).color(new float[]{0, 1, 1}).build()))
                .build();
+        when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
        when(dictionaryClient.getAllTypes()).thenReturn(typeResponse);
-        DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
-                .build();
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
-        DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
-                .build();
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor());
-        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
-            entityRedactionService.processDocument(classifiedDoc);
-            assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
-            assertThat(classifiedDoc.getEntities().get(1)).hasSize(5); // 4 out of 5 entities recognized on page 1
-        }
    }


@@ -0,0 +1,67 @@
+package com.iqser.red.service.redaction.v1.server.segmentation;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+
+@SpringBootTest
+@RunWith(SpringRunner.class)
+public class PdfSegmentationServiceTest {
+
+    @Autowired
+    private PdfSegmentationService pdfSegmentationService;
+
+    @Autowired
+    private RulingCleaningService rulingCleaningService;
+
+    @Autowired
+    private TableExtractionService tableExtractionService;
+
+    @Autowired
+    private BlockificationService blockificationService;
+
+    @MockBean
+    private KieContainer kieContainer;
+
+
+    @Test
+    public void testPDFSegmentationWithComplexTable() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
+
+        try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
+            Document document = pdfSegmentationService.parseDocument(pdDocument);
+            assertThat(document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())).isNotEmpty();
+            Table table = document.getParagraphs()
+                    .stream()
+                    .flatMap(paragraph -> paragraph.getTables().stream())
+                    .collect(Collectors.toList())
+                    .get(0);
+            assertThat(table.getColCount()).isEqualTo(6);
+            assertThat(table.getRowCount()).isEqualTo(13);
+            assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
+        }
+    }
+
+}
@@ -49,64 +49,69 @@ rule "5: Do not redact in guideline sections"
 		section.redactNot("address", 5, "Section is a guideline section.");
 	end

-rule "6: Redact if must redact entry is found"
-	when
-		eval(section.contains("must_redact")==true);
-	then
-		section.redact("name", 6, "must_redact entry was found.");
-		section.redact("address", 6, "must_redact entry was found.");
-	end
-
-
-rule "7: Redact contact information, if applicant is found"
+rule "6: Redact contact information, if applicant is found"
 	when
 		eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant"));
 	then
-		section.redactLineAfter("Name:", "address", 7, "Applicant information was found");
-		section.redactBetween("Address:", "Contact", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Contact point:",  "address", 7, "Applicant information was found");
-		section.redactLineAfter("Phone:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Fax:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Tel.:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Tel:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("E-mail:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Email:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Contact:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Telephone number:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Fax number:", "address", 7, "Applicant information was found");
-		section.redactLineAfter("Telephone:", "address", 7, "Applicant information was found");
-		section.redactBetween("No:", "Fax", "address", 7, "Applicant information was found");
-		section.redactBetween("Contact:", "Tel.:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Name:", "address", 6, "Applicant information was found");
+		section.redactBetween("Address:", "Contact", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Contact point:",  "address", 6, "Applicant information was found");
+		section.redactLineAfter("Phone:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Fax:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Tel.:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Tel:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("E-mail:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Email:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Contact:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Telephone number:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Fax number:", "address", 6, "Applicant information was found");
+		section.redactLineAfter("Telephone:", "address", 6, "Applicant information was found");
+		section.redactBetween("No:", "Fax", "address", 6, "Applicant information was found");
+		section.redactBetween("Contact:", "Tel.:", "address", 6, "Applicant information was found");
 	end

-rule "8: Redact contact information, if Producer is found"
+rule "7: Redact contact information, if Producer is found"
 	when
 		eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance"));
 	then
-		section.redactLineAfter("Name:", "address", 8, "Producer was found");
-		section.redactBetween("Address:", "Contact", "address", 8, "Producer was found");
-		section.redactBetween("Contact:", "Phone", "address", 8, "Producer was found");
-		section.redactBetween("Contact:", "Telephone number:", "address", 8, "Producer was found");
-		section.redactBetween("Address:", "Manufacturing", "address", 8, "Producer was found");
-		section.redactLineAfter("Telephone:", "address", 8, "Producer was found");
-		section.redactLineAfter("Phone:", "address", 8, "Producer was found");
-		section.redactLineAfter("Fax:", "address", 8, "Producer was found");
-		section.redactLineAfter("E-mail:", "address", 8, "Producer was found");
-		section.redactLineAfter("Contact:", "address", 8, "Producer was found");
-		section.redactLineAfter("Fax number:", "address", 8, "Producer was found");
-		section.redactLineAfter("Telephone number:", "address", 8, "Producer was found");
-		section.redactLineAfter("Tel:", "address", 8, "Producer was found");
-		section.redactBetween("No:", "Fax", "address", 8, "Producer was found");
+		section.redactLineAfter("Name:", "address", 7, "Producer was found");
+		section.redactBetween("Address:", "Contact", "address", 7, "Producer was found");
+		section.redactBetween("Contact:", "Phone", "address", 7, "Producer was found");
+		section.redactBetween("Contact:", "Telephone number:", "address", 7, "Producer was found");
+		section.redactBetween("Address:", "Manufacturing", "address", 7, "Producer was found");
+		section.redactLineAfter("Telephone:", "address", 7, "Producer was found");
+		section.redactLineAfter("Phone:", "address", 7, "Producer was found");
+		section.redactLineAfter("Fax:", "address", 7, "Producer was found");
+		section.redactLineAfter("E-mail:", "address", 7, "Producer was found");
+		section.redactLineAfter("Contact:", "address", 7, "Producer was found");
+		section.redactLineAfter("Fax number:", "address", 7, "Producer was found");
+		section.redactLineAfter("Telephone number:", "address", 7, "Producer was found");
+		section.redactLineAfter("Tel:", "address", 7, "Producer was found");
+		section.redactBetween("No:", "Fax", "address", 7, "Producer was found");
 	end

-rule "9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
+rule "8: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study"
    when
-        Section(tabularData != null && tabularData.size() > 0
-            && tabularData.containsKey("Vertebrate study Y/N")
-            && tabularData.get("Vertebrate study Y/N").equals("Y")
-        )
+        Section(isVertebrateStudy())
    then
-        section.redact("name", 9, "Redacted because row is a vertebrate study");
-        section.redact("address", 9, "Redacted because rows is a vertebrate study");
+        section.redact("name", 8, "Redacted because row is a vertebrate study");
+        section.redact("address", 8, "Redacted because row is a vertebrate study");
        section.highlightCell("Vertebrate study Y/N", 9);
-    end
+    end
+
+rule "9: Not redacted because Vertebrate Study = N"
+    when
+        Section(isNotVertebrateStudy())
+    then
+        section.redactNot("name", 9, "Not redacted because row is not a vertebrate study");
+        section.redactNot("address", 9, "Not redacted because row is not a vertebrate study");
+    end
+
+
+rule "10: Redact if must redact entry is found"
+	when
+		eval(section.contains("must_redact")==true);
+	then
+		section.redact("name", 10, "must_redact entry was found.");
+		section.redact("address", 10, "must_redact entry was found.");
+	end
Author	SHA1	Message	Date
Thierry Göckel	76369f13f8	Remove redundant comment	2020-08-18 20:53:41 +02:00
Thierry Göckel	1fff4f7eb0	RED-101: Add features as requested in PR - Add rule for non-vertebrate studies - Don't highlight ambiguous cell values - Rewrite rules in more human-readable fashion	2020-08-18 20:35:40 +02:00
Thierry Göckel	c7f5b4a280	Add unit test for table structure requirements	2020-08-18 20:35:17 +02:00
Thierry Göckel	4236fa05cc	Make table structure detection more robust	2020-08-13 17:48:37 +02:00
Thierry Goeckel	031f10435d	Pull request #24 : RED-88: Enabled manual redaction Merge in RED/redaction-service from RED-88 to master * commit '5d286f58e918b22f041ae54f982f14b2d4fcefa6': PR Improvements redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java online editiert mit Bitbucket redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java online editiert mit Bitbucket RED-88: Enabled manual redaction	2020-08-13 16:20:53 +02:00
deiflaender	5d286f58e9	PR Improvements	2020-08-13 16:10:22 +02:00
Dominique Eiflaender	5d18a46ec3	redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java online editiert mit Bitbucket	2020-08-13 16:07:11 +02:00
Dominique Eiflaender	9b61971132	redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java online editiert mit Bitbucket	2020-08-13 16:07:05 +02:00
deiflaender	2b93ae57d5	RED-88: Enabled manual redaction	2020-08-13 15:34:09 +02:00
Thierry Goeckel	954765759c	Pull request #23 : Log warning message if tabular data mismatches Merge in RED/redaction-service from RED-101-quickfix to master * commit '17aabcd09c1f76bdd467f66aac0cf243c0625734': Fix index out of bounds exception Remove redundant warn message Fix NPE for empty cells Add test redacting all files and expecting no exception Log warning message if tabular data mismatches	2020-08-13 11:33:17 +02:00
Thierry Göckel	17aabcd09c	Fix index out of bounds exception	2020-08-13 11:13:01 +02:00
Thierry Göckel	32aa500983	Remove redundant warn message	2020-08-13 11:04:56 +02:00
Thierry Göckel	a151a13b4c	Fix NPE for empty cells	2020-08-13 11:02:09 +02:00
Thierry Göckel	5542a97a38	Add test redacting all files and expecting no exception	2020-08-13 10:14:49 +02:00
Thierry Göckel	8edaa93bda	Log warning message if tabular data mismatches	2020-08-13 10:07:53 +02:00
Dominique Eiflaender	98e0bf5606	Pull request #22 : RED-242: Return sectionNumber for tests Merge in RED/redaction-service from RED-242 to master * commit '8344b7ccafcf98e7376c53a0a79708c51daa6d0a': RED-242: Return sectionNumber for tests	2020-08-12 15:51:50 +02:00
deiflaender	8344b7ccaf	RED-242: Return sectionNumber for tests	2020-08-12 15:46:33 +02:00