Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
*some refactors
*some comment cleanup
This commit is contained in:
Kilian Schuettler 2023-04-03 11:52:32 +02:00 committed by Kilian Schuettler
parent d9c27d70d1
commit 6f8242a55e
13 changed files with 123 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java
View File

@ -13,7 +13,9 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DocumentData {
List<PageData> pages;
List<AtomicTextBlockData> atomicTextBlocks;
TableOfContentsData tableOfContents;
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java
View File

@ -11,7 +11,9 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class PageData {
int number;
int height;
int width;
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/PropertiesMapper.java
View File

@ -1,2 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.data.mapper;public class PropertiesMapper {
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java
View File

@ -54,7 +54,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class DocumentGraphFactory {
public static final double TABLE_CELL_MERGE_SIZE_THRESHOLD = 0.05;
public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
private final ImageSortService imageSortService;
@ -216,7 +216,7 @@ public class DocumentGraphFactory {
private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) {
return cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java
View File

@ -88,7 +88,7 @@ public class RectangleTransformations {
@Override
public BiConsumer<Area, Rectangle2D> accumulator() {
return (a, b) -> a.add(new Area(b));
return (area, rectangle2D) -> area.add(new Area(rectangle2D));
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java
View File

@ -49,13 +49,12 @@ public class SearchTextWithTextPositionFactory {
if (isLineBreak(currentTextPosition, previousTextPosition)) {
if (stringIdx - lastHyphenIdx < 3) {
sb.replace(lastHyphenIdx, sb.length(), "");
sb.delete(lastHyphenIdx, sb.length());
stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx);
stringIdx = lastHyphenIdx;
lastHyphenIdx = -3;
}
lineBreaksStringIdx.add(stringIdx);
}
if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextPositionOperations.java
View File

@ -8,14 +8,12 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
public class TextPositionOperations {
public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextBlock> textBlocks) {
return textBlocks
.stream()
.flatMap(tb -> tb.getSequences().stream())
return textBlocks.stream()//
.flatMap(tb -> tb.getSequences().stream())//
.sorted(Comparator.comparingDouble(TextPositionSequence::getMaxYDirAdj)//
.thenComparing(TextPositionSequence::getMaxXDirAdj))
.thenComparing(TextPositionSequence::getMaxXDirAdj))//
.toList();
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java
View File

@ -8,9 +8,10 @@ public enum NodeType {
}
},
HEADLINE {
public String toString() {
return "Headline";
}
public String toString() {
return "Headline";
}
},
PARAGRAPH {
public String toString() {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java
View File

@ -159,7 +159,7 @@ public interface SemanticNode {
/**
* @param string A String
* @param string A String which the TextBlock might contain
* @return true, if this node's TextBlock contains the string
*/
default boolean containsString(String string) {
@ -169,7 +169,7 @@ public interface SemanticNode {
/**
* THis function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
* It sets the fields accordingly and recursively calls this function on all its children.
*
* @param entity EntityNode, which is being inserted into the graph

45
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.data.mapper;
package com.iqser.red.service.redaction.v1.server.document.mapper;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
@ -14,7 +14,6 @@ import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsDa
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
@ -54,9 +53,9 @@ public class DocumentDataMapper {
}
Map<String, String> properties = switch (entry.type()) {
case TABLE -> buildTableProperties((TableNode) entry.node());
case TABLE_CELL -> buildTableCellProperties((TableCellNode) entry.node());
case IMAGE -> buildImageProperties((ImageNode) entry.node());
case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.node());
case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.node());
case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.node());
default -> new HashMap<>();
};
@ -72,42 +71,6 @@ public class DocumentDataMapper {
}
private Map<String, String> buildImageProperties(ImageNode image) {
Map<String, String> properties = new HashMap<>();
properties.put("imageType", image.getImageType().toString());
properties.put("transparency", String.valueOf(image.isTransparency()));
properties.put("position", RectangleTransformations.toString(image.getPosition()));
return properties;
}
private Map<String, String> buildTableCellProperties(TableCellNode tableCell) {
Map<String, String> properties = new HashMap<>();
properties.put("row", String.valueOf(tableCell.getRow()));
properties.put("col", String.valueOf(tableCell.getCol()));
properties.put("header", String.valueOf(tableCell.isHeader()));
if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
throw new IllegalArgumentException("TableCell can only occur on a single page!");
}
String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
properties.put("bBox", bBoxString);
return properties;
}
private Map<String, String> buildTableProperties(TableNode table) {
Map<String, String> properties = new HashMap<>();
properties.put("numberOfRows", String.valueOf(table.getNumberOfRows()));
properties.put("numberOfCols", String.valueOf(table.getNumberOfCols()));
return properties;
}
private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java
View File

@ -1,6 +1,8 @@
package com.iqser.red.service.redaction.v1.server.document.data.mapper;
package com.iqser.red.service.redaction.v1.server.document.mapper;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.parseRectangle2D;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageProperties;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableCellProperties;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableProperties;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
@ -37,7 +39,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.Atomic
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
@Service
public class DocumentGraphMapper {
@ -114,29 +115,25 @@ public class DocumentGraphMapper {
private ImageNode buildImage(Context context, Map<String, String> properties) {
ImageType imageType = parseImageType(properties.get("imageType"));
boolean transparency = Boolean.parseBoolean(properties.get("transparency"));
Rectangle2D position = parseRectangle2D(properties.get("position"));
return ImageNode.builder().tableOfContents(context.tableOfContents()).imageType(imageType).transparency(transparency).position(position).build();
var builder = ImageNode.builder();
parseImageProperties(properties, builder);
return builder.tableOfContents(context.tableOfContents()).build();
}
private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
int row = Integer.parseInt(properties.get("row"));
int col = Integer.parseInt(properties.get("col"));
boolean header = Boolean.parseBoolean(properties.get("header"));
Rectangle2D bBox = parseRectangle2D(properties.get("bBox"));
return TableCellNode.builder().terminal(terminal).bBox(bBox).header(header).col(col).row(row).tableOfContents(context.tableOfContents()).build();
TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder();
parseTableCellProperties(properties, builder);
return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build();
}
private TableNode buildTable(Context context, Map<String, String> properties) {
int numberOfRows = Integer.parseInt(properties.get("numberOfRows"));
int numberOfCols = Integer.parseInt(properties.get("numberOfCols"));
return TableNode.builder().tableOfContents(context.tableOfContents()).numberOfRows(numberOfRows).numberOfCols(numberOfCols).build();
TableNode.TableNodeBuilder builder = TableNode.builder();
parseTableProperties(properties, builder);
return TableNode.builder().tableOfContents(context.tableOfContents()).build();
}
@ -165,18 +162,6 @@ public class DocumentGraphMapper {
}
private static ImageType parseImageType(String imageType) {
return switch (imageType) {
case "LOGO" -> ImageType.LOGO;
case "FORMULA" -> ImageType.FORMULA;
case "SIGNATURE" -> ImageType.SIGNATURE;
case "OCR" -> ImageType.OCR;
default -> ImageType.OTHER;
};
}
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds)

87
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java Normal file
View File

@ -0,0 +1,87 @@
package com.iqser.red.service.redaction.v1.server.document.mapper;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.parseRectangle2D;
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
public class PropertiesMapper {
public static Map<String, String> buildImageProperties(ImageNode image) {
Map<String, String> properties = new HashMap<>();
properties.put("imageType", image.getImageType().toString());
properties.put("transparency", String.valueOf(image.isTransparency()));
properties.put("position", RectangleTransformations.toString(image.getPosition()));
return properties;
}
public static Map<String, String> buildTableCellProperties(TableCellNode tableCell) {
Map<String, String> properties = new HashMap<>();
properties.put("row", String.valueOf(tableCell.getRow()));
properties.put("col", String.valueOf(tableCell.getCol()));
properties.put("header", String.valueOf(tableCell.isHeader()));
if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
throw new IllegalArgumentException("TableCell can only occur on a single page!");
}
String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
properties.put("bBox", bBoxString);
return properties;
}
public static Map<String, String> buildTableProperties(TableNode table) {
Map<String, String> properties = new HashMap<>();
properties.put("numberOfRows", String.valueOf(table.getNumberOfRows()));
properties.put("numberOfCols", String.valueOf(table.getNumberOfCols()));
return properties;
}
public static void parseImageProperties(Map<String, String> properties, ImageNode.ImageNodeBuilder builder) {
builder.imageType(parseImageType(properties.get("imageType")));
builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
builder.position(parseRectangle2D(properties.get("position")));
}
public static void parseTableCellProperties(Map<String, String> properties, TableCellNode.TableCellNodeBuilder builder) {
builder.row(Integer.parseInt(properties.get("row")));
builder.col(Integer.parseInt(properties.get("col")));
builder.header(Boolean.parseBoolean(properties.get("header")));
builder.bBox(parseRectangle2D(properties.get("bBox")));
}
public static void parseTableProperties(Map<String, String> properties, TableNode.TableNodeBuilder builder) {
builder.numberOfRows(Integer.parseInt(properties.get("numberOfRows")));
builder.numberOfCols(Integer.parseInt(properties.get("numberOfCols")));
}
private static ImageType parseImageType(String imageType) {
return switch (imageType) {
case "LOGO" -> ImageType.LOGO;
case "FORMULA" -> ImageType.FORMULA;
case "SIGNATURE" -> ImageType.SIGNATURE;
case "OCR" -> ImageType.OCR;
default -> ImageType.OTHER;
};
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java
View File

@ -6,9 +6,9 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;