RSS-165: Added rules to redactCellBelow

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -39,6 +39,10 @@ public class SectionText {
     @Builder.Default
     private List<Integer> cellStarts = new ArrayList<>();
 
+    private List<List<CellValue>> tableValues;
+    private List<CellValue> rowValues;
+    private int rowIndex;
+
 
     public void setTabularData(Map<String, CellValue> tabularData) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -54,8 +54,9 @@ public class BlockificationService {
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -107,51 +108,51 @@ public class BlockificationService {
             chunkBlockList1.add(cb1);
         }
 
-        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+//        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
-
+//
-        TextBlock previousLeft = null;
+//        TextBlock previousLeft = null;
-        TextBlock previousRight = null;
+//        TextBlock previousRight = null;
-        while (itty.hasNext()) {
+//        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
+//            TextBlock block = (TextBlock) itty.next();
-
+//
-            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
+//            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
+//                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                    previousLeft.add(block);
+//                    previousLeft.add(block);
-                    itty.remove();
+//                    itty.remove();
-                    continue;
+//                    continue;
-                }
+//                }
-            }
+//            }
-
+//
-            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
+//            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
+//                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                    previousRight.add(block);
+//                    previousRight.add(block);
-                    itty.remove();
+//                    itty.remove();
-                    continue;
+//                    continue;
-                }
+//                }
-            }
+//            }
-
+//
-            if (block.getOrientation().equals(Orientation.LEFT)) {
+//            if (block.getOrientation().equals(Orientation.LEFT)) {
-                previousLeft = block;
+//                previousLeft = block;
-            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+//            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
-                previousRight = block;
+//                previousRight = block;
-            }
+//            }
-        }
+//        }
-
+//
-        itty = chunkBlockList1.iterator();
+//        itty = chunkBlockList1.iterator();
-        TextBlock previous = null;
+//        TextBlock previous = null;
-        while (itty.hasNext()) {
+//        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
+//            TextBlock block = (TextBlock) itty.next();
-
+//
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
+//            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
-                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
+//                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
+//                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
-                previous.add(block);
+//                previous.add(block);
-                itty.remove();
+//                itty.remove();
-                continue;
+//                continue;
-            }
+//            }
-
+//
-            previous = block;
+//            previous = block;
-        }
+//        }
 
         return new Page(chunkBlockList1);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -84,7 +84,8 @@ public class BodyTextFrameService {
                     }
 
                     float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < 2.9f) {
+                    System.out.println("PageNr: " + page.getPageNumber() + " Height " + page.getPageHeight() + " Width: " +page.getPageWidth() + " Rotation: " +page.getRotation());
+                    if (approxLineCount < 2.9f && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10)) {
                         continue;
                     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,15 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
 import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 
 import lombok.RequiredArgsConstructor;
@@ -52,11 +56,32 @@ public class ClassificationService {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z]{2,50}", true);
+        var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+        var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
+
+        Matcher matcher = pattern.matcher(textBlock.toString());
+        Matcher matcher2 = pattern2.matcher(textBlock.toString());
+        Matcher matcher3 = pattern3.matcher(textBlock.toString());
+
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification("Other");
             return;
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (textBlock.getText().length() > 5 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
+
+                && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
+            textBlock.setClassification("H 1");
+            document.setHeadlines(true);
+
+        }
+        else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.matches() && !matcher3.matches()) {
+            textBlock.setClassification("H 2");
+            document.setHeadlines(true);
+        }
+        else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Header");
 
@@ -69,32 +94,36 @@ public class ClassificationService {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification("Title");
             }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+        }
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+//        else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+//                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                    textBlock.setClassification("H " + i);
+//                .getCountPerValue()
-                    document.setHeadlines(true);
+//                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                }
+//                .get(0)
-            }
+//                .getTextPositions()
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                .get(0)
-                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-                .getMostPopular()
+//
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                .get(0)
+//                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                .getTextPositions()
+//                    textBlock.setClassification("H " + i);
-                .get(0)
+//                    document.setHeadlines(true);
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//                }
-            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            }
-            document.setHeadlines(true);
+//        }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//        else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//                .get(0)
+//                .getTextPositions()
+//                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            document.setHeadlines(true);
+//        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
                 .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
             textBlock.setClassification("TextBlock Bold");
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -18,6 +18,8 @@ import java.util.Set;
 public class Entity implements ReasonHolder {
 
     private String word;
+
+    @EqualsAndHashCode.Include
     private String type;
     private boolean redaction;
     private boolean falsePositive;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/OffsetString.java

@@ -23,5 +23,17 @@ public class OffsetString {
         return new OffsetString(trimmed, newStart, newEnd);
     }
 
+
+    public OffsetString replaceAll(String regex, String replacement) {
+
+        String trimmed = this.value.replaceAll(regex, replacement);
+        int indexInUntrimmed = this.value.indexOf(trimmed);
+
+        int newStart = this.start + indexInUntrimmed;
+        int newEnd = newStart + trimmed.length();
+
+        return new OffsetString(trimmed, newStart, newEnd);
+    }
+
 }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -6,6 +6,7 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextPositionSequenceComparator;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -13,6 +14,8 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import org.apache.pdfbox.util.QuickSort;
+
 import lombok.Getter;
 
 public class SearchableText {
@@ -118,7 +121,7 @@ public class SearchableText {
                 } else {
                     counter = 0;
                     if (!crossSequenceParts.isEmpty()) {
-                        j--;
+                        j=j-partMatch.length() - 1;
                     }
                     crossSequenceParts = new ArrayList<>();
                     partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
@@ -216,16 +219,13 @@ public class SearchableText {
 
     public String getAsStringWithLinebreaksSorted(List<TextPositionSequence> sequences) {
 
-        var sorted = sequences.stream()
+        var quickSorted = new ArrayList<>(sequences);
-                .sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj()))
+        QuickSort.sort(quickSorted, new TextPositionSequenceComparator());
-                .sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj()))
-                .sorted(Comparator.comparing(a -> a.getPage()))
-                .collect(Collectors.toList());
 
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;
-        for (TextPositionSequence word : sorted) {
+        for (TextPositionSequence word : quickSorted) {
 
             if (previous != null) {
                 if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -5,6 +5,7 @@ import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -32,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 
 import lombok.Builder;
 import lombok.Data;
@@ -85,6 +87,10 @@ public class Section {
 
     private boolean isInTable;
 
+    private List<List<CellValue>> table;
+    private List<CellValue> row;
+    private int currentRowIndex;
+
 
     @Deprecated
     @SuppressWarnings("unused")
@@ -207,13 +213,17 @@ public class Section {
         return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) {
+    public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String... values) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue()));
+        var valueSet = new HashSet<>(Arrays.asList(values));
+
+        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && valueSet.contains(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
@@ -247,7 +257,7 @@ public class Section {
             fileAttributes = new ArrayList<>();
         }
 
-        boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel().equals(label) && f.getValue().equals(value));
+        boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel() != null && f.getLabel().equals(label) && f.getValue() != null && f.getValue().equals(value));
 
         if (!exists) {
             fileAttributes.add(FileAttribute.builder().label(label).value(value).build());
@@ -274,7 +284,7 @@ public class Section {
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match)) {
-                boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel().equals(label) && f.getValue().equals(match));
+                boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel() != null && f.getLabel().equals(label) && f.getValue() != null && f.getValue().equals(match));
 
                 if (!exists) {
                     fileAttributes.add(FileAttribute.builder().label(label).value(match).build());
@@ -293,6 +303,27 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName);
     }
 
+    @WhenCondition
+    public boolean hasCellAbove(@Argument(ArgumentType.STRING) String aboveValue) {
+
+        if(row != null) {
+
+            for (int i = 0; i < row.size(); i++) {
+
+                int t = 0;
+                for (List<CellValue> tableRow : table) {
+                    if (currentRowIndex > t && tableRow.get(i).toString().equals(aboveValue)) {
+                        return true;
+                    }
+                    t++;
+                }
+            }
+        }
+        return false;
+    }
+
+
+
 
     @SuppressWarnings("unused")
     @WhenCondition
@@ -554,6 +585,35 @@ public class Section {
     }
 
 
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
+                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                          @Argument(ArgumentType.INTEGER) int group,
+                                          @Argument(ArgumentType.TYPE) String asType,
+                                          @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                          @Argument(ArgumentType.STRING) String reason,
+                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
+                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                          @Argument(ArgumentType.INTEGER) int group,
+                                          @Argument(ArgumentType.TYPE) String asType,
+                                          @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                          @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch,
+                                          @Argument(ArgumentType.STRING) String reason,
+                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false, onlyExactMatch);
+    }
+
+
     @ThenAction
     @SuppressWarnings("unused")
     public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@@ -564,21 +624,7 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false, false);
-    }
-
-
-    @ThenAction
-    @SuppressWarnings("unused")
-    public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                              @Argument(ArgumentType.INTEGER) int group,
-                              @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                              @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
-
-        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
     }
 
 
@@ -593,7 +639,23 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, onlyExactMatch);
     }
 
 
@@ -606,7 +668,7 @@ public class Section {
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, false);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, false, false);
     }
 
 
@@ -621,7 +683,7 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false, false, false);
     }
 
 
@@ -636,7 +698,7 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false, false, false);
     }
 
 
@@ -665,7 +727,9 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, false);
+                sortedResult,
+                false,
+                false);
     }
 
 
@@ -697,10 +761,46 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, ignoreTables);
+                sortedResult,
+                ignoreTables,
+                false);
     }
 
 
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start,
+                              @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                              @Argument(ArgumentType.BOOLEAN) boolean ignoreTables,
+                              @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch) {
+
+        redactBetween(start,
+                stop,
+                includeStart,
+                includeStop,
+                asType,
+                ruleNumber,
+                redactEverywhere,
+                excludeHeadLine,
+                reason,
+                legalBasis,
+                true,
+                skipRemoveEntitiesContainedInLarger,
+                sortedResult,
+                ignoreTables,
+                onlyExactMatch);
+    }
+
 
     @ThenAction
     @SuppressWarnings("unused")
@@ -729,7 +829,9 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, false);
+                sortedResult,
+                false,
+                false);
     }
 
 
@@ -747,7 +849,8 @@ public class Section {
                                      @Argument(ArgumentType.STRING) String reason,
                                      @Argument(ArgumentType.STRING) String legalBasis,
                                      @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
-                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                                     @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch) {
 
         String startValue = getFirstRexExMatch(searchText, startPattern, startPatternCaseInsensitive, startGroup);
 
@@ -772,11 +875,46 @@ public class Section {
                     legalBasis,
                     true,
                     skipRemoveEntitiesContainedInLarger,
-                    sortedResult, false);
+                    sortedResult,
+                    false,
+                    onlyExactMatch);
         }
     }
 
 
+    @ThenAction
+    public void redactBetweenRegexes(@Argument(ArgumentType.STRING) String startPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean startPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int startGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                                     @Argument(ArgumentType.STRING) String stopPattern,
+                                     @Argument(ArgumentType.BOOLEAN) boolean stopPatternCaseInsensitive,
+                                     @Argument(ArgumentType.INTEGER) int stopGroup,
+                                     @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                                     @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                     @Argument(ArgumentType.TYPE) String type,
+                                     @Argument(ArgumentType.STRING) String reason,
+                                     @Argument(ArgumentType.STRING) String legalBasis,
+                                     @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+
+        redactBetweenRegexes(startPattern,
+        startPatternCaseInsensitive,
+        startGroup,
+        includeStart,
+       stopPattern,
+        stopPatternCaseInsensitive,
+        stopGroup,
+        includeStop,
+        ruleNumber,
+        type,
+        reason,
+        legalBasis,
+        skipRemoveEntitiesContainedInLarger,
+        sortedResult, false);
+    }
+
+
     @Deprecated
     @ThenAction
     @SuppressWarnings("unused")
@@ -787,19 +925,7 @@ public class Section {
                                  @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start,
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, null, false, false, false, false, false);
-                stop,
-                false,
-                false,
-                asType,
-                ruleNumber,
-                redactEverywhere,
-                false,
-                reason,
-                null,
-                false,
-                false,
-                false, false);
     }
 
 
@@ -813,19 +939,7 @@ public class Section {
                                  @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start,
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false, false, false, false);
-                stop,
-                false,
-                false,
-                asType,
-                ruleNumber,
-                redactEverywhere,
-                excludeHeadLine,
-                reason,
-                null,
-                false,
-                false,
-                false, false);
     }
 
 
@@ -1347,6 +1461,98 @@ public class Section {
     }
 
 
+    public void redactCellBelow(int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis, String... aboveValues) {
+
+        Set<String> aboveValueSet = Set.of(aboveValues);
+
+        if(row != null){
+
+            for (int i = 0; i< row.size(); i++) {
+
+                int t = 0;
+                Set<String> matches = new HashSet<>();
+                for (List<CellValue> tableRow : table){
+                    if(currentRowIndex > t && aboveValueSet.contains(tableRow.get(i).toString())){
+                        matches.add(tableRow.get(i).toString());
+                        if(matches.size() == aboveValueSet.size()){
+
+                            CellValue value = row.get(i);
+                            if (value == null) {
+//                                log.warn("Could not find any data for {}.", cellHeader);
+                            } else {
+                                String word = value.toString();
+
+                                Entity entity = new Entity(word,
+                                        type,
+                                        value.getRowSpanStart(),
+                                        value.getRowSpanStart() + word.length(),
+                                        headline,
+                                        sectionNumber,
+                                        false,
+                                        false,
+                                        Engine.RULE,
+                                        EntityType.ENTITY);
+                                entity.setRedaction(redact);
+                                entity.setMatchedRule(ruleNumber);
+                                entity.setRedactionReason(reason);
+                                entity.setTargetSequences(value.getTextBlocks()
+                                        .stream()
+                                        .map(TextBlock::getSequences)
+                                        .flatMap(Collection::stream)
+                                        .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
+                                entity.setLegalBasis(legalBasis);
+
+                                Set<Entity> singleEntitySet = new HashSet<>();
+                                singleEntitySet.add(entity);
+
+                                EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
+                                EntitySearchUtils.removeFalsePositives(singleEntitySet,
+                                        searchText,
+                                        dictionary.getType(type),
+                                        new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
+
+                                if (!singleEntitySet.isEmpty()) {
+                                    EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
+
+                                    EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+
+                                    if (addAsRecommendations && !isLocal()) {
+                                        String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+                                        Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                                        Matcher matcher = pattern.matcher(cleanedWord);
+
+                                        while (matcher.find()) {
+                                            String match = matcher.group().trim();
+                                            if (match.length() >= 3) {
+                                                localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
+                                                String lastname = match.split(" ")[0];
+                                                localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+
+
+
+
+
+
+                        }
+
+
+                    }
+                    t++;
+                }
+
+            }
+        }
+
+
+    }
+
+
+
     private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
 
         String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
@@ -1445,7 +1651,8 @@ public class Section {
                 if (StringUtils.isNotBlank(stringOffset.getValue())) {
                     var trimmedOffsetString = stringOffset.trim();
                     Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
-                            .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet());
+                            .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
+                            .collect(Collectors.toSet());
                     found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
 
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
@@ -1459,7 +1666,16 @@ public class Section {
     }
 
 
-    private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
+    private void redactByRegExWithNewlines(String pattern,
+                                           boolean patternCaseInsensitive,
+                                           int group,
+                                           String asType,
+                                           int ruleNumber,
+                                           String reason,
+                                           String legalBasis,
+                                           boolean redaction,
+                                           boolean skipRemoveEntitiesContainedInLarger,
+                                           boolean onlyExactMatch) {
 
         Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive);
 
@@ -1467,8 +1683,16 @@ public class Section {
 
         while (matcher.find()) {
             String match = matcher.group(group);
+            int start = matcher.start(group);
+            int end = matcher.end(group);
+
+            OffsetString offsetString = new OffsetString(match, start, end);
+            var trimmedOffsetString = offsetString.replaceAll("\\n", " ").trim();
+
             if (StringUtils.isNotBlank(match)) {
-                Set<Entity> found = findEntities(match.replaceAll("\\n", " ").trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
+                        .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
+                        .collect(Collectors.toSet());
                 found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
@@ -1476,7 +1700,16 @@ public class Section {
     }
 
 
-    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
+    private void redactByRegEx(String pattern,
+                               boolean patternCaseInsensitive,
+                               int group,
+                               String asType,
+                               int ruleNumber,
+                               String reason,
+                               String legalBasis,
+                               boolean redaction,
+                               boolean skipRemoveEntitiesContainedInLarger,
+                               boolean onlyExactMatch) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1484,8 +1717,17 @@ public class Section {
 
         while (matcher.find()) {
             String match = matcher.group(group);
+
+            int start = matcher.start(group);
+            int end = matcher.end(group);
+
+            OffsetString offsetString = new OffsetString(match, start, end);
+            var trimmedOffsetString = offsetString.trim();
+
             if (StringUtils.isNotBlank(match)) {
-                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
+                        .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
+                        .collect(Collectors.toSet());
                 found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
@@ -1522,60 +1764,65 @@ public class Section {
                                boolean redaction,
                                boolean skipRemoveEntitiesContainedInLarger,
                                boolean sortedResult,
-                               boolean ignoreTables) {
+                               boolean ignoreTables,
+                               boolean onlyExactMatch) {
 
-
+        if (isInTable && ignoreTables) {
-        if(isInTable && ignoreTables){
             return;
         }
 
-        String[] values = new String[1];
+        List<OffsetString> values = new ArrayList<>();
 
         if (start.isEmpty() && stop.isEmpty()) {
             if (excludeHeadLine && searchText.contains(headline)) {
-                values[0] = StringUtils.substringAfter(searchText, headline);
+                values.add(OffsetStringUtils.substringAfter(searchText, headline));
             } else {
-                values[0] = searchText;
+                values.add(new OffsetString(searchText, 0, searchText.length()));
             }
         } else if (start.isEmpty() && searchText.contains(stop)) {
-            values[0] = StringUtils.substringBefore(searchText, stop);
+            values.add(OffsetStringUtils.substringBefore(searchText, stop));
         } else if (stop.isEmpty() && searchText.contains(start)) {
-            values[0] = StringUtils.substringAfter(searchText, start);
+            values.add(OffsetStringUtils.substringAfter(searchText, start));
         } else {
-            values = StringUtils.substringsBetween(searchText, start, stop);
+            var stringsBetween = OffsetStringUtils.substringsBetween(searchText, start, stop);
+            if(stringsBetween != null) {
+                values.addAll(stringsBetween);
+            }
         }
 
-        if (values != null) {
+        for (OffsetString value : values) {
-            for (String value : values) {
+            if (StringUtils.isNotBlank(value.getValue())) {
-                if (StringUtils.isNotBlank(value)) {
 
-                    String searchString = value;
+                OffsetString searchString = value;
 
-                    if (!start.isEmpty() && includeStart) {
+                if (!start.isEmpty() && includeStart) {
-                        searchString = start + searchString;
+                    searchString = new OffsetString(start + searchString.getValue(), searchString.getStart() - start.length(), searchString.getEnd());
+                }
+
+                if (!stop.isEmpty() && includeStop) {
+                    searchString = new OffsetString(searchString.getValue() + stop, searchString.getStart(), searchString.getEnd() + stop.length());
+                }
+
+                var trimmedOffsetString = searchString.trim();
+
+                Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
+                        .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
+                        .collect(Collectors.toSet());
+                found.forEach(f -> {
+                    f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
+                    if (sortedResult) {
+                        f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences()
+                                .stream()
+                                .map(EntityPositionSequence::getSequences)
+                                .flatMap(Collection::stream)
+                                .collect(Collectors.toList())));
                     }
+                });
 
-                    if (!stop.isEmpty() && includeStop) {
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
-                        searchString = searchString + stop;
-                    }
 
-                    Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                if (redactEverywhere && !isLocal()) {
-                    found.forEach(f -> {
+                    localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(trimmedOffsetString.getValue());
-                        f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
-                        if (sortedResult) {
-                            f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences()
-                                    .stream()
-                                    .map(EntityPositionSequence::getSequences)
-                                    .flatMap(Collection::stream)
-                                    .collect(Collectors.toList())));
-                        }
-                    });
-
-                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
-
-                    if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(searchString.trim());
-                    }
                 }
             }
         }
@@ -1646,6 +1893,7 @@ public class Section {
                 entity.setMatchedRule(ruleNumber);
                 entity.setRedactionReason(reason);
                 entity.setLegalBasis(legalBasis);
+                entity.getEngines().add(Engine.RULE);
             });
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -137,6 +137,9 @@ public class EntityRedactionService {
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
 
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                    .table(reanalysisSection.getTableValues())
+                    .row(reanalysisSection.getRowValues())
+                    .currentRowIndex(reanalysisSection.getRowIndex())
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
                     .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -7,6 +7,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.Point;
@@ -92,7 +93,7 @@ public class RedactionLogCreatorService {
         List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 
         // Duplicates can exist due table extraction columns over multiple rows.
-        Set<String> processedIds = new HashSet<>();
+        Set<Pair<String, String>> processedIds = new HashSet<>();
 
         entityLoop:
         for (Entity entity : entities.get(page)) {
@@ -100,12 +101,11 @@ public class RedactionLogCreatorService {
             for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
 
                 RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
-                if (processedIds.contains(entityPositionSequence.getId())) {
+                if (processedIds.contains(Pair.of(entityPositionSequence.getId(), entity.getType()))) {
 
-                    // TODO refactor this outer loop jump as soon as we have the time.
                     continue entityLoop;
                 } else {
-                    processedIds.add(entityPositionSequence.getId());
+                    processedIds.add(Pair.of(entityPositionSequence.getId(), entity.getType()));
                 }
 
                 redactionLogEntry.setId(entityPositionSequence.getId());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -152,10 +152,10 @@ public class RedactionLogMergeService {
                     return true;
                 }
             }
-            if (processedIds.contains(entry.getId())) {
+//            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
+//                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
+//                return true;
-            }
+//            }
             processedIds.add(entry.getId());
             return false;
         });

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -75,6 +75,9 @@ public class SectionTextBuilderService {
         List<SectionText> sectionTexts = new ArrayList<>();
         boolean hasHeader = hasTableHeader(table);
 
+
+        List<List<CellValue>> tableCellValues = new ArrayList<>();
+        int i = 0;
         for (List<Cell> row : table.getRows()) {
 
             List<TextBlock> textBlocks = new ArrayList<>();
@@ -83,6 +86,7 @@ public class SectionTextBuilderService {
             List<Integer> startOffsets = new ArrayList<>();
 
             int startOffset = 0;
+            List<CellValue> rowCellValues = new ArrayList<>();
             for (int cellNum = 0; cellNum < row.size(); cellNum++) {
 
                 Cell cell = row.get(cellNum);
@@ -100,12 +104,14 @@ public class SectionTextBuilderService {
                         tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset));
                     }
                 }
+                rowCellValues.add(new CellValue(cell.getTextBlocks(), startOffset));
 
                 textBlocks.addAll(cell.getTextBlocks());
 
                 startOffsets.add(startOffset);
                 startOffset = startOffset + cell.toString().trim().length() + 1;
             }
+            tableCellValues.add(rowCellValues);
 
             sectionTexts.add(SectionText.builder()
                     .text(getRowText(textBlocks))
@@ -116,9 +122,13 @@ public class SectionTextBuilderService {
                     .cellStarts(startOffsets)
                     .textBlocks(textBlocks)
                     .sectionAreas(areas)
+                            .tableValues(tableCellValues)
+                            .rowValues(rowCellValues)
+                            .rowIndex(i)
                     .build());
 
             sectionNumber.incrementAndGet();
+            i++;
         }
 
         return sectionTexts;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,16 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
@@ -24,9 +37,12 @@ public class EntitySearchUtils {
 
     public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details) {
 
-        Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+        if (type != null) {
-        markFalsePositives(found, falsePositives);
+
-        found.removeIf(f -> f.isFalsePositive());
+            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+            markFalsePositives(found, falsePositives);
+            found.removeIf(f -> f.isFalsePositive());
+        }
     }
 
 
@@ -34,12 +50,15 @@ public class EntitySearchUtils {
 
         Set<Entity> found = find(inputString, searchImplementation, details);
 
-        if (details.getEntityType() == EntityType.RECOMMENDATION) {
+        if(type != null) {
-            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
+
-            markFalsePositives(found, falseRecommendations);
+            if (details.getEntityType() == EntityType.RECOMMENDATION) {
-        } else {
+                Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+                markFalsePositives(found, falseRecommendations);
-            markFalsePositives(found, falsePositives);
+            } else {
+                Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+                markFalsePositives(found, falsePositives);
+            }
         }
 
         return found;
@@ -267,7 +286,7 @@ public class EntitySearchUtils {
                         existing.setRedaction(true);
                     }
                 }
-            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
+            } else if (dictionary.getDictionaryRank(existing.getType()) < dictionary.getDictionaryRank(found.getType())) {
                 entities.remove(found);
                 entities.add(found);
             }
@@ -292,9 +311,12 @@ public class EntitySearchUtils {
                 .get(0)
                 .getSequences()
                 .get(0)
-                .getMinXDirAdj() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxXDirAdj() && image.getPosition()
+                .getMinXDirAdj() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences()
-                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getMinYDirAdj() && image.getPosition().getY() + image.getPosition()
+                .get(0)
-                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxYDirAdj();
+                .getSequences()
+                .get(0)
+                .getMaxXDirAdj() && image.getPosition().getY() < entity.getPositionSequences().get(0).getSequences().get(0).getMinYDirAdj() && image.getPosition()
+                .getY() + image.getPosition().getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxYDirAdj();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java

@@ -15,8 +15,8 @@ public class OffsetStringUtils {
     /**
      * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
      *
-     * @param str – the String containing the substrings, null returns null, empty returns empty
+     * @param str   – the String containing the substrings, null returns null, empty returns empty
-     * @param open – the String identifying the start of the substring, empty returns null
+     * @param open  – the String identifying the start of the substring, empty returns null
      * @param close – the String identifying the end of the substring, empty returns null
      * @return a list of Strings with their offsets
      */
@@ -52,5 +52,41 @@ public class OffsetStringUtils {
         return list;
     }
 
+
+    public static OffsetString substringAfter(final String str, final String separator) {
+
+        if (StringUtils.isEmpty(str)) {
+            return new OffsetString(str, 0, 0);
+        }
+        if (separator == null) {
+            return new OffsetString(str, 0, 0);
+        }
+        final int pos = str.indexOf(separator);
+        if (pos == StringUtils.INDEX_NOT_FOUND) {
+            return new OffsetString(str, 0, 0);
+        }
+
+        String result = str.substring(pos + separator.length());
+
+        return new OffsetString(result, pos + separator.length(), str.length());
+    }
+
+
+    public static OffsetString substringBefore(final String str, final String separator) {
+
+        if (StringUtils.isEmpty(str) || separator == null) {
+            return new OffsetString(str, 0, 0);
+        }
+        if (separator.isEmpty()) {
+            return new OffsetString(str, 0, 0);
+        }
+        final int pos = str.indexOf(separator);
+        if (pos == StringUtils.INDEX_NOT_FOUND) {
+            return new OffsetString(str, 0, 0);
+        }
+        String result = str.substring(0, pos);
+        return new OffsetString(result, 0, pos);
+    }
+
 }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextPositionSequenceComparator.java

@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
+
+    @Override
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
+        // only compare text that is in the same direction
+        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
+        if (cmp1 != 0) {
+            return cmp1;
+        }
+
+        int page = Integer.compare(pos1.getPage(), pos2.getPage());
+        if (page != 0){
+            return page;
+        }
+
+        // get the text direction adjusted coordinates
+        float x1 = pos1.getMinXDirAdj();
+        float x2 = pos2.getMinXDirAdj();
+
+        float pos1YBottom = pos1.getMaxYDirAdj();
+        float pos2YBottom = pos2.getMaxYDirAdj();
+
+        // note that the coordinates have been adjusted so 0,0 is in upper left
+        float pos1YTop = pos1YBottom - pos1.getHeight();
+        float pos2YTop = pos2YBottom - pos2.getHeight();
+
+        float yDifference = Math.abs(pos1YBottom - pos2YBottom);
+
+        // we will do a simple tolerance comparison
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
+            return Float.compare(x1, x2);
+        } else if (pos1YBottom < pos2YBottom) {
+            return -1;
+        } else {
+            return 1;
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -92,6 +92,7 @@ public class PdfSegmentationService {
 
                     PDFLinesTextStripper stripper = new PDFLinesTextStripper();
                     PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                    stripper.setSortByPosition(true);
                     stripper.setPageNumber(pageNumber);
                     stripper.setStartPage(pageNumber);
                     stripper.setEndPage(pageNumber);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -68,7 +68,7 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -1,13 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 
@@ -261,30 +265,58 @@ public class Table extends AbstractTextContainer {
         if (cells.isEmpty()) {
             return rv;
         }
-        cells.sort(Comparator.comparingDouble(Rectangle::getLeft));
 
-        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(), 2))));
+        Set<Float> uniqueX = new HashSet<>();
+        cells.forEach(c -> uniqueX.add(c.getLeft()));
+        cells.forEach(c -> uniqueX.add(c.getRight()));
 
-        Iterator<Cell> iter = cells.iterator();
+        Set<Float> uniqueY = new HashSet<>();
-        Cell c = iter.next();
+        cells.forEach(c -> uniqueY.add(c.getBottom()));
-        float lastTop = c.getBottom();
+        cells.forEach(c -> uniqueY.add(c.getTop()));
-        List<Cell> lastRow = new ArrayList<>();
-        lastRow.add(c);
-        rv.add(lastRow);
 
-        while (iter.hasNext()) {
+        var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList());
-            c = iter.next();
+        var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList());
-            if (!Utils.feq(c.getBottom(), lastTop)) {
+
-                lastRow = new ArrayList<>();
+        List<List<Cell>> matrix = new ArrayList<>();
-                rv.add(lastRow);
+
+        Float prevY = null;
+        var uniqueYIterator = sortedUniqueY.iterator();
+
+        while (uniqueYIterator.hasNext()) {
+
+            List<Cell> row = new ArrayList<>();
+
+            var y = uniqueYIterator.next();
+            var uniqueXIterator = sortedUniqueX.iterator();
+            Float prevX = null;
+            while (uniqueXIterator.hasNext()) {
+                var x = uniqueXIterator.next();
+
+                if (prevY != null && prevX != null) {
+                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
+
+                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    if (intersectionCell.isPresent()) {
+                        cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
+                    }
+                    row.add(cell);
+                }
+                prevX = x;
             }
-            lastRow.add(c);
+
-            lastTop = c.getBottom();
+            if (prevY != null && prevX != null) {
+                matrix.add(row);
+            }
+            prevY = y;
         }
-        return rv;
+
+        Collections.reverse(matrix);
+
+        return matrix;
     }
 
 
+
     @Override
     public String getText() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -4,6 +4,7 @@ import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -51,9 +52,14 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.ChangeType;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@@ -61,6 +67,7 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -133,6 +140,37 @@ public class HeadlinesGoldStandardIntegrationTest {
     private final static String TEST_FILE_ID = "123";
 
 
+    @Test
+    public void extractHeadlines() throws IOException {
+
+        AnalyzeRequest request = prepareStorage("files/RSS/26 - Sedaxane - Acute Oral Toxicity - Rat.pdf");
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Headlines.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+
+        RedactionRequest redactionRequest = RedactionRequest.builder()
+                .dossierId(request.getDossierId())
+                .fileId(request.getFileId())
+                .dossierTemplateId(request.getDossierTemplateId())
+                .build();
+
+        RedactionResult result1 = redactionController.classify(redactionRequest);
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/HeadlinesClassified.pdf")) {
+            fileOutputStream.write(result1.getDocument());
+        }
+    }
+
+
     @Test
     public void testHeadlineDetection() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/RSS/06 - Isopyrazam - Acute Oral Toxicity Rat.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);