RSS-165: Added rules to redactCellBelow

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -39,6 +39,10 @@ public class SectionText {
     @Builder.Default
     private List<Integer> cellStarts = new ArrayList<>();
 
+    private List<List<CellValue>> tableValues;
+    private List<CellValue> rowValues;
+    private int rowIndex;
+
 
     public void setTabularData(Map<String, CellValue> tabularData) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -33,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 
 import lombok.Builder;
 import lombok.Data;
@@ -86,6 +87,10 @@ public class Section {
 
     private boolean isInTable;
 
+    private List<List<CellValue>> table;
+    private List<CellValue> row;
+    private int currentRowIndex;
+
 
     @Deprecated
     @SuppressWarnings("unused")
@@ -298,6 +303,27 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName);
     }
 
+    @WhenCondition
+    public boolean hasCellAbove(@Argument(ArgumentType.STRING) String aboveValue) {
+
+        if(row != null) {
+
+            for (int i = 0; i < row.size(); i++) {
+
+                int t = 0;
+                for (List<CellValue> tableRow : table) {
+                    if (currentRowIndex > t && tableRow.get(i).toString().equals(aboveValue)) {
+                        return true;
+                    }
+                    t++;
+                }
+            }
+        }
+        return false;
+    }
+
+
+
 
     @SuppressWarnings("unused")
     @WhenCondition
@@ -1435,6 +1461,98 @@ public class Section {
     }
 
 
+    public void redactCellBelow(int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis, String... aboveValues) {
+
+        Set<String> aboveValueSet = Set.of(aboveValues);
+
+        if(row != null){
+
+            for (int i = 0; i< row.size(); i++) {
+
+                int t = 0;
+                Set<String> matches = new HashSet<>();
+                for (List<CellValue> tableRow : table){
+                    if(currentRowIndex > t && aboveValueSet.contains(tableRow.get(i).toString())){
+                        matches.add(tableRow.get(i).toString());
+                        if(matches.size() == aboveValueSet.size()){
+
+                            CellValue value = row.get(i);
+                            if (value == null) {
+//                                log.warn("Could not find any data for {}.", cellHeader);
+                            } else {
+                                String word = value.toString();
+
+                                Entity entity = new Entity(word,
+                                        type,
+                                        value.getRowSpanStart(),
+                                        value.getRowSpanStart() + word.length(),
+                                        headline,
+                                        sectionNumber,
+                                        false,
+                                        false,
+                                        Engine.RULE,
+                                        EntityType.ENTITY);
+                                entity.setRedaction(redact);
+                                entity.setMatchedRule(ruleNumber);
+                                entity.setRedactionReason(reason);
+                                entity.setTargetSequences(value.getTextBlocks()
+                                        .stream()
+                                        .map(TextBlock::getSequences)
+                                        .flatMap(Collection::stream)
+                                        .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
+                                entity.setLegalBasis(legalBasis);
+
+                                Set<Entity> singleEntitySet = new HashSet<>();
+                                singleEntitySet.add(entity);
+
+                                EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
+                                EntitySearchUtils.removeFalsePositives(singleEntitySet,
+                                        searchText,
+                                        dictionary.getType(type),
+                                        new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
+
+                                if (!singleEntitySet.isEmpty()) {
+                                    EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
+
+                                    EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+
+                                    if (addAsRecommendations && !isLocal()) {
+                                        String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+                                        Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                                        Matcher matcher = pattern.matcher(cleanedWord);
+
+                                        while (matcher.find()) {
+                                            String match = matcher.group().trim();
+                                            if (match.length() >= 3) {
+                                                localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
+                                                String lastname = match.split(" ")[0];
+                                                localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+
+
+
+
+
+
+                        }
+
+
+                    }
+                    t++;
+                }
+
+            }
+        }
+
+
+    }
+
+
+
     private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
 
         String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -137,6 +137,9 @@ public class EntityRedactionService {
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
 
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                    .table(reanalysisSection.getTableValues())
+                    .row(reanalysisSection.getRowValues())
+                    .currentRowIndex(reanalysisSection.getRowIndex())
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
                     .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -75,6 +75,9 @@ public class SectionTextBuilderService {
         List<SectionText> sectionTexts = new ArrayList<>();
         boolean hasHeader = hasTableHeader(table);
 
+
+        List<List<CellValue>> tableCellValues = new ArrayList<>();
+        int i = 0;
         for (List<Cell> row : table.getRows()) {
 
             List<TextBlock> textBlocks = new ArrayList<>();
@@ -83,6 +86,7 @@ public class SectionTextBuilderService {
             List<Integer> startOffsets = new ArrayList<>();
 
             int startOffset = 0;
+            List<CellValue> rowCellValues = new ArrayList<>();
             for (int cellNum = 0; cellNum < row.size(); cellNum++) {
 
                 Cell cell = row.get(cellNum);
@@ -100,12 +104,14 @@ public class SectionTextBuilderService {
                         tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset));
                     }
                 }
+                rowCellValues.add(new CellValue(cell.getTextBlocks(), startOffset));
 
                 textBlocks.addAll(cell.getTextBlocks());
 
                 startOffsets.add(startOffset);
                 startOffset = startOffset + cell.toString().trim().length() + 1;
             }
+            tableCellValues.add(rowCellValues);
 
             sectionTexts.add(SectionText.builder()
                     .text(getRowText(textBlocks))
@@ -116,9 +122,13 @@ public class SectionTextBuilderService {
                     .cellStarts(startOffsets)
                     .textBlocks(textBlocks)
                     .sectionAreas(areas)
+                            .tableValues(tableCellValues)
+                            .rowValues(rowCellValues)
+                            .rowIndex(i)
                     .build());
 
             sectionNumber.incrementAndGet();
+            i++;
         }
 
         return sectionTexts;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -1,13 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 
@@ -261,29 +265,57 @@ public class Table extends AbstractTextContainer {
         if (cells.isEmpty()) {
             return rv;
         }
-        cells.sort(Comparator.comparingDouble(Rectangle::getLeft));
 
-        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(), 2))));
+        Set<Float> uniqueX = new HashSet<>();
+        cells.forEach(c -> uniqueX.add(c.getLeft()));
+        cells.forEach(c -> uniqueX.add(c.getRight()));
 
-        Iterator<Cell> iter = cells.iterator();
+        Set<Float> uniqueY = new HashSet<>();
-        Cell c = iter.next();
+        cells.forEach(c -> uniqueY.add(c.getBottom()));
-        float lastTop = c.getBottom();
+        cells.forEach(c -> uniqueY.add(c.getTop()));
-        List<Cell> lastRow = new ArrayList<>();
-        lastRow.add(c);
-        rv.add(lastRow);
 
-        while (iter.hasNext()) {
+        var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList());
-            c = iter.next();
+        var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList());
-            if (!Utils.feq(c.getBottom(), lastTop)) {
+
-                lastRow = new ArrayList<>();
+        List<List<Cell>> matrix = new ArrayList<>();
-                rv.add(lastRow);
+
+        Float prevY = null;
+        var uniqueYIterator = sortedUniqueY.iterator();
+
+        while (uniqueYIterator.hasNext()) {
+
+            List<Cell> row = new ArrayList<>();
+
+            var y = uniqueYIterator.next();
+            var uniqueXIterator = sortedUniqueX.iterator();
+            Float prevX = null;
+            while (uniqueXIterator.hasNext()) {
+                var x = uniqueXIterator.next();
+
+                if (prevY != null && prevX != null) {
+                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
+
+                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    if (intersectionCell.isPresent()) {
+                        cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
                     }
-            lastRow.add(c);
+                    row.add(cell);
-            lastTop = c.getBottom();
                 }
-        return rv;
+                prevX = x;
             }
 
+            if (prevY != null && prevX != null) {
+                matrix.add(row);
+            }
+            prevY = y;
+        }
+
+        Collections.reverse(matrix);
+
+        return matrix;
+    }
+
+
 
     @Override
     public String getText() {