Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-165: Added rules to redactCellBelow

Browse Source
This commit is contained in:
deiflaender 2022-12-12 13:52:29 +01:00
parent 6c8868dd43
commit b860d26c39
5 changed files with 183 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java
View File

@ -39,6 +39,10 @@ public class SectionText {
@Builder.Default
private List<Integer> cellStarts = new ArrayList<>();
private List<List<CellValue>> tableValues;
private List<CellValue> rowValues;
private int rowIndex;
public void setTabularData(Map<String, CellValue> tabularData) {

118
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -33,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import lombok.Builder;
import lombok.Data;
@ -86,6 +87,10 @@ public class Section {
private boolean isInTable;
private List<List<CellValue>> table;
private List<CellValue> row;
private int currentRowIndex;
@Deprecated
@SuppressWarnings("unused")
@ -298,6 +303,27 @@ public class Section {
return tabularData != null && tabularData.containsKey(cleanHeaderName);
}
@WhenCondition
public boolean hasCellAbove(@Argument(ArgumentType.STRING) String aboveValue) {
if(row != null) {
for (int i = 0; i < row.size(); i++) {
int t = 0;
for (List<CellValue> tableRow : table) {
if (currentRowIndex > t && tableRow.get(i).toString().equals(aboveValue)) {
return true;
}
t++;
}
}
}
return false;
}
@SuppressWarnings("unused")
@WhenCondition
@ -1435,6 +1461,98 @@ public class Section {
}
public void redactCellBelow(int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis, String... aboveValues) {
Set<String> aboveValueSet = Set.of(aboveValues);
if(row != null){
for (int i = 0; i< row.size(); i++) {
int t = 0;
Set<String> matches = new HashSet<>();
for (List<CellValue> tableRow : table){
if(currentRowIndex > t && aboveValueSet.contains(tableRow.get(i).toString())){
matches.add(tableRow.get(i).toString());
if(matches.size() == aboveValueSet.size()){
CellValue value = row.get(i);
if (value == null) {
// log.warn("Could not find any data for {}.", cellHeader);
} else {
String word = value.toString();
Entity entity = new Entity(word,
type,
value.getRowSpanStart(),
value.getRowSpanStart() + word.length(),
headline,
sectionNumber,
false,
false,
Engine.RULE,
EntityType.ENTITY);
entity.setRedaction(redact);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(reason);
entity.setTargetSequences(value.getTextBlocks()
.stream()
.map(TextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
entity.setLegalBasis(legalBasis);
Set<Entity> singleEntitySet = new HashSet<>();
singleEntitySet.add(entity);
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
EntitySearchUtils.removeFalsePositives(singleEntitySet,
searchText,
dictionary.getType(type),
new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
if (!singleEntitySet.isEmpty()) {
EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
Matcher matcher = pattern.matcher(cleanedWord);
while (matcher.find()) {
String match = matcher.group().trim();
if (match.length() >= 3) {
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
String lastname = match.split(" ")[0];
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
}
}
}
}
}
}
}
t++;
}
}
}
}
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -137,6 +137,9 @@ public class EntityRedactionService {
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
.table(reanalysisSection.getTableValues())
.row(reanalysisSection.getRowValues())
.currentRowIndex(reanalysisSection.getRowIndex())
.isLocal(false)
.dictionaryTypes(dictionary.getTypes())
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
View File

@ -75,6 +75,9 @@ public class SectionTextBuilderService {
List<SectionText> sectionTexts = new ArrayList<>();
boolean hasHeader = hasTableHeader(table);
List<List<CellValue>> tableCellValues = new ArrayList<>();
int i = 0;
for (List<Cell> row : table.getRows()) {
List<TextBlock> textBlocks = new ArrayList<>();
@ -83,6 +86,7 @@ public class SectionTextBuilderService {
List<Integer> startOffsets = new ArrayList<>();
int startOffset = 0;
List<CellValue> rowCellValues = new ArrayList<>();
for (int cellNum = 0; cellNum < row.size(); cellNum++) {
Cell cell = row.get(cellNum);
@ -100,12 +104,14 @@ public class SectionTextBuilderService {
tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset));
}
}
rowCellValues.add(new CellValue(cell.getTextBlocks(), startOffset));
textBlocks.addAll(cell.getTextBlocks());
startOffsets.add(startOffset);
startOffset = startOffset + cell.toString().trim().length() + 1;
}
tableCellValues.add(rowCellValues);
sectionTexts.add(SectionText.builder()
.text(getRowText(textBlocks))
@ -116,9 +122,13 @@ public class SectionTextBuilderService {
.cellStarts(startOffsets)
.textBlocks(textBlocks)
.sectionAreas(areas)
.tableValues(tableCellValues)
.rowValues(rowCellValues)
.rowIndex(i)
.build());
sectionNumber.incrementAndGet();
i++;
}
return sectionTexts;

64
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java
View File

@ -1,13 +1,17 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
import java.awt.geom.Point2D;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
@ -261,30 +265,58 @@ public class Table extends AbstractTextContainer {
if (cells.isEmpty()) {
return rv;
}
cells.sort(Comparator.comparingDouble(Rectangle::getLeft));
cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(), 2))));
Set<Float> uniqueX = new HashSet<>();
cells.forEach(c -> uniqueX.add(c.getLeft()));
cells.forEach(c -> uniqueX.add(c.getRight()));
Iterator<Cell> iter = cells.iterator();
Cell c = iter.next();
float lastTop = c.getBottom();
List<Cell> lastRow = new ArrayList<>();
lastRow.add(c);
rv.add(lastRow);
Set<Float> uniqueY = new HashSet<>();
cells.forEach(c -> uniqueY.add(c.getBottom()));
cells.forEach(c -> uniqueY.add(c.getTop()));
while (iter.hasNext()) {
c = iter.next();
if (!Utils.feq(c.getBottom(), lastTop)) {
lastRow = new ArrayList<>();
rv.add(lastRow);
var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList());
var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList());
List<List<Cell>> matrix = new ArrayList<>();
Float prevY = null;
var uniqueYIterator = sortedUniqueY.iterator();
while (uniqueYIterator.hasNext()) {
List<Cell> row = new ArrayList<>();
var y = uniqueYIterator.next();
var uniqueXIterator = sortedUniqueX.iterator();
Float prevX = null;
while (uniqueXIterator.hasNext()) {
var x = uniqueXIterator.next();
if (prevY != null && prevX != null) {
var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
if (intersectionCell.isPresent()) {
cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
}
row.add(cell);
}
prevX = x;
}
lastRow.add(c);
lastTop = c.getBottom();
if (prevY != null && prevX != null) {
matrix.add(row);
}
prevY = y;
}
return rv;
Collections.reverse(matrix);
return matrix;
}
@Override
public String getText() {