diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java index cb7d1416..f48d4c7a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java @@ -39,6 +39,10 @@ public class SectionText { @Builder.Default private List cellStarts = new ArrayList<>(); + private List> tableValues; + private List rowValues; + private int rowIndex; + public void setTabularData(Map tabularData) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 528cbe8f..72e1f868 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -33,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import lombok.Builder; import lombok.Data; @@ -86,6 +87,10 @@ public class Section { private boolean isInTable; + private List> table; + private List row; + private int currentRowIndex; + @Deprecated @SuppressWarnings("unused") @@ -298,6 +303,27 @@ public class Section { return tabularData != null && tabularData.containsKey(cleanHeaderName); } + @WhenCondition + public boolean hasCellAbove(@Argument(ArgumentType.STRING) String aboveValue) { + + if(row != null) { + + for (int i = 0; i < row.size(); i++) { + + int t = 0; + for (List tableRow : table) { + if (currentRowIndex > t && tableRow.get(i).toString().equals(aboveValue)) { + return true; + } + t++; + } + } + } + return false; + } + + + @SuppressWarnings("unused") @WhenCondition @@ -1435,6 +1461,98 @@ public class Section { } + public void redactCellBelow(int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis, String... aboveValues) { + + Set aboveValueSet = Set.of(aboveValues); + + if(row != null){ + + for (int i = 0; i< row.size(); i++) { + + int t = 0; + Set matches = new HashSet<>(); + for (List tableRow : table){ + if(currentRowIndex > t && aboveValueSet.contains(tableRow.get(i).toString())){ + matches.add(tableRow.get(i).toString()); + if(matches.size() == aboveValueSet.size()){ + + CellValue value = row.get(i); + if (value == null) { +// log.warn("Could not find any data for {}.", cellHeader); + } else { + String word = value.toString(); + + Entity entity = new Entity(word, + type, + value.getRowSpanStart(), + value.getRowSpanStart() + word.length(), + headline, + sectionNumber, + false, + false, + Engine.RULE, + EntityType.ENTITY); + entity.setRedaction(redact); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setTargetSequences(value.getTextBlocks() + .stream() + .map(TextBlock::getSequences) + .flatMap(Collection::stream) + .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted + entity.setLegalBasis(legalBasis); + + Set singleEntitySet = new HashSet<>(); + singleEntitySet.add(entity); + + EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions); + EntitySearchUtils.removeFalsePositives(singleEntitySet, + searchText, + dictionary.getType(type), + new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY)); + + if (!singleEntitySet.isEmpty()) { + EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary); + + EntitySearchUtils.removeEntitiesContainedInLarger(entities); + + if (addAsRecommendations && !isLocal()) { + String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; + Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER; + Matcher matcher = pattern.matcher(cleanedWord); + + while (matcher.find()) { + String match = matcher.group().trim(); + if (match.length() >= 3) { + localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match); + String lastname = match.split(" ")[0]; + localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname); + } + } + } + } + } + + + + + + + } + + + } + t++; + } + + } + } + + + } + + + private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) { String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 55a6f1a2..faa3ff23 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -137,6 +137,9 @@ public class EntityRedactionService { log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages()); sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() + .table(reanalysisSection.getTableValues()) + .row(reanalysisSection.getRowValues()) + .currentRowIndex(reanalysisSection.getRowIndex()) .isLocal(false) .dictionaryTypes(dictionary.getTypes()) .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java index b8cd232b..d92513d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java @@ -75,6 +75,9 @@ public class SectionTextBuilderService { List sectionTexts = new ArrayList<>(); boolean hasHeader = hasTableHeader(table); + + List> tableCellValues = new ArrayList<>(); + int i = 0; for (List row : table.getRows()) { List textBlocks = new ArrayList<>(); @@ -83,6 +86,7 @@ public class SectionTextBuilderService { List startOffsets = new ArrayList<>(); int startOffset = 0; + List rowCellValues = new ArrayList<>(); for (int cellNum = 0; cellNum < row.size(); cellNum++) { Cell cell = row.get(cellNum); @@ -100,12 +104,14 @@ public class SectionTextBuilderService { tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset)); } } + rowCellValues.add(new CellValue(cell.getTextBlocks(), startOffset)); textBlocks.addAll(cell.getTextBlocks()); startOffsets.add(startOffset); startOffset = startOffset + cell.toString().trim().length() + 1; } + tableCellValues.add(rowCellValues); sectionTexts.add(SectionText.builder() .text(getRowText(textBlocks)) @@ -116,9 +122,13 @@ public class SectionTextBuilderService { .cellStarts(startOffsets) .textBlocks(textBlocks) .sectionAreas(areas) + .tableValues(tableCellValues) + .rowValues(rowCellValues) + .rowIndex(i) .build()); sectionNumber.incrementAndGet(); + i++; } return sectionTexts; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java index 62910062..b0278eae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java @@ -1,13 +1,17 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model; +import java.awt.geom.Point2D; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -261,30 +265,58 @@ public class Table extends AbstractTextContainer { if (cells.isEmpty()) { return rv; } - cells.sort(Comparator.comparingDouble(Rectangle::getLeft)); - cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(), 2)))); + Set uniqueX = new HashSet<>(); + cells.forEach(c -> uniqueX.add(c.getLeft())); + cells.forEach(c -> uniqueX.add(c.getRight())); - Iterator iter = cells.iterator(); - Cell c = iter.next(); - float lastTop = c.getBottom(); - List lastRow = new ArrayList<>(); - lastRow.add(c); - rv.add(lastRow); + Set uniqueY = new HashSet<>(); + cells.forEach(c -> uniqueY.add(c.getBottom())); + cells.forEach(c -> uniqueY.add(c.getTop())); - while (iter.hasNext()) { - c = iter.next(); - if (!Utils.feq(c.getBottom(), lastTop)) { - lastRow = new ArrayList<>(); - rv.add(lastRow); + var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList()); + var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList()); + + List> matrix = new ArrayList<>(); + + Float prevY = null; + var uniqueYIterator = sortedUniqueY.iterator(); + + while (uniqueYIterator.hasNext()) { + + List row = new ArrayList<>(); + + var y = uniqueYIterator.next(); + var uniqueXIterator = sortedUniqueX.iterator(); + Float prevX = null; + while (uniqueXIterator.hasNext()) { + var x = uniqueXIterator.next(); + + if (prevY != null && prevX != null) { + var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y)); + + var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst(); + if (intersectionCell.isPresent()) { + cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks()); + } + row.add(cell); + } + prevX = x; } - lastRow.add(c); - lastTop = c.getBottom(); + + if (prevY != null && prevX != null) { + matrix.add(row); + } + prevY = y; } - return rv; + + Collections.reverse(matrix); + + return matrix; } + @Override public String getText() {