RED-6009: Document Tree Structure
* refactored package structure
This commit is contained in:
Kilian Schuettler
parent
9c48d36b05
commit
1c0f7e308b
@ -1,8 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
public enum Orientation {
|
||||
|
||||
NONE,
|
||||
LEFT,
|
||||
RIGHT
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
@ -10,11 +10,11 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -22,26 +22,26 @@ import lombok.SneakyThrows;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageService {
|
||||
public class ImageServiceResponseAdapter {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId) {
|
||||
public Map<Integer, List<ClassifiedImage>> convertImages(String dossierId, String fileId) {
|
||||
|
||||
var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
|
||||
|
||||
ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);
|
||||
|
||||
Map<Integer, List<PdfImage>> images = new HashMap<>();
|
||||
Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
|
||||
imageServiceResponse.getData().forEach(imageMetadata -> {
|
||||
var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
|
||||
.getLabel()
|
||||
.toUpperCase(Locale.ROOT)) : ImageType.OTHER;
|
||||
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
|
||||
.add(new ClassifiedImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
|
||||
imageMetadata.getPosition().getY1(),
|
||||
imageMetadata.getGeometry().getWidth(),
|
||||
imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
|
||||
@ -53,7 +53,7 @@ public class ImageService {
|
||||
.getLabel()
|
||||
.toUpperCase(Locale.ROOT)) : ImageType.OTHER;
|
||||
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
|
||||
.add(new ClassifiedImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
|
||||
imageMetadata.getPosition().getY1(),
|
||||
imageMetadata.getGeometry().getWidth(),
|
||||
imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
@ -10,9 +10,9 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableCells;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TableService {
|
||||
public class TableServiceResponseAdapter {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
@ -1,9 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class ImageServiceResponse {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Probability {
|
||||
|
||||
private boolean unconfident;
|
||||
|
||||
}
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.table;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.table;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.table;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ -1,9 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.table;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ -1,9 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
@ -12,7 +11,7 @@ import lombok.NoArgsConstructor;
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public abstract class AbstractTextContainer {
|
||||
public abstract class AbstractPageBlock {
|
||||
|
||||
@JsonIgnore
|
||||
protected float minX;
|
||||
@ -40,7 +39,7 @@ public abstract class AbstractTextContainer {
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(AbstractTextContainer other) {
|
||||
public boolean contains(AbstractPageBlock other) {
|
||||
|
||||
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
|
||||
}
|
||||
@ -67,7 +66,7 @@ public abstract class AbstractTextContainer {
|
||||
}
|
||||
|
||||
|
||||
public boolean intersectsY(AbstractTextContainer atc) {
|
||||
public boolean intersectsY(AbstractPageBlock atc) {
|
||||
|
||||
return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
|
||||
}
|
||||
@ -1,9 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
|
||||
|
||||
import lombok.Data;
|
||||
@ -1,6 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
@ -9,6 +7,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class FloatFrequencyCounter {
|
||||
|
||||
@Getter
|
||||
@ -1,13 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Footer {
|
||||
@ -1,13 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Header {
|
||||
@ -0,0 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
public enum Orientation {
|
||||
|
||||
NONE,
|
||||
LEFT,
|
||||
RIGHT
|
||||
}
|
||||
@ -1,11 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
@ -16,9 +16,9 @@ import lombok.RequiredArgsConstructor;
|
||||
public class Page {
|
||||
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
private List<AbstractPageBlock> textBlocks;
|
||||
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private List<ClassifiedImage> images = new ArrayList<>();
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
@ -15,8 +15,8 @@ import lombok.NoArgsConstructor;
|
||||
@NoArgsConstructor
|
||||
public class Section implements Comparable {
|
||||
|
||||
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private List<AbstractPageBlock> pageBlocks = new ArrayList<>();
|
||||
private List<ClassifiedImage> images = new ArrayList<>();
|
||||
private String headline;
|
||||
|
||||
|
||||
@ -61,4 +61,5 @@ public class Section implements Comparable {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,4 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
@ -6,7 +9,7 @@ import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class PdfImage {
|
||||
public class ClassifiedImage {
|
||||
|
||||
@NonNull
|
||||
private RedRectangle2D position;
|
||||
@ -1,18 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Value;
|
||||
@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class CleanRulings {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
@ -1,14 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Formatter;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.CohenSutherlandClipping;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@SuppressWarnings("all")
|
||||
@ -1,9 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
@ -12,14 +11,15 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class Table extends AbstractTextContainer {
|
||||
public class Table extends AbstractPageBlock {
|
||||
|
||||
private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
@ -7,15 +7,6 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
@ -1,5 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class StringFrequencyCounter {
|
||||
|
||||
@Getter
|
||||
@ -1,13 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@ -1,13 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@ -18,7 +16,7 @@ import lombok.NoArgsConstructor;
|
||||
@Builder
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class TextBlock extends AbstractTextContainer {
|
||||
public class TextBlock extends AbstractPageBlock {
|
||||
|
||||
@Builder.Default
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
@ -97,6 +95,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the maxX value in pdf coordinate system.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
@ -177,6 +176,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
|
||||
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
|
||||
|
||||
this.indexOnPage = indexOnPage;
|
||||
this.minX = minX;
|
||||
this.maxX = maxX;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
@ -1,4 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
@ -6,8 +8,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class UnclassifiedText {
|
||||
@ -14,34 +14,18 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
import java.util.WeakHashMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.fontbox.ttf.TrueTypeFont;
|
||||
import org.apache.fontbox.util.BoundingBox;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFStreamEngine;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType3Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.apache.pdfbox.util.Vector;
|
||||
import org.apache.pdfbox.contentstream.operator.DrawObject;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Concatenate;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Restore;
|
||||
@ -50,22 +34,36 @@ import org.apache.pdfbox.contentstream.operator.state.SetGraphicsStateParameters
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetMatrix;
|
||||
import org.apache.pdfbox.contentstream.operator.text.BeginText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.EndText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveTextSetLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.NextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetCharSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRenderingMode;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRise;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetWordSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType3Font;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
|
||||
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.apache.pdfbox.util.Vector;
|
||||
|
||||
/**
|
||||
* LEGACY text calculations which are known to be incorrect but are depended on by PDFTextStripper.
|
||||
@ -1,17 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
public class PDFAreaTextStripper extends PDFTextStripperByArea {
|
||||
|
||||
@Getter
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
@ -34,9 +34,9 @@ import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
@ -14,7 +14,7 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import static java.util.stream.Collectors.toSet;
|
||||
|
||||
@ -9,15 +9,15 @@ import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("all")
|
||||
@ -40,7 +40,7 @@ public class BlockificationService {
|
||||
|
||||
int indexOnPage = 0;
|
||||
List<TextPositionSequence> chunkWords = new ArrayList<>();
|
||||
List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
|
||||
List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
|
||||
|
||||
float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
|
||||
TextPositionSequence prev = null;
|
||||
@ -111,7 +111,7 @@ public class BlockificationService {
|
||||
chunkBlockList1.add(cb1);
|
||||
}
|
||||
|
||||
Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
|
||||
Iterator<AbstractPageBlock> itty = chunkBlockList1.iterator();
|
||||
|
||||
TextBlock previousLeft = null;
|
||||
TextBlock previousRight = null;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@ -6,13 +6,13 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
|
||||
|
||||
@Service
|
||||
public class BodyTextFrameService {
|
||||
@ -75,7 +75,7 @@ public class BodyTextFrameService {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (AbstractTextContainer container : page.getTextBlocks()) {
|
||||
for (AbstractPageBlock container : page.getTextBlocks()) {
|
||||
|
||||
if (container instanceof TextBlock) {
|
||||
TextBlock textBlock = (TextBlock) container;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
@ -6,11 +6,11 @@ import java.util.regex.Pattern;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -40,7 +40,7 @@ public class ClassificationService {
|
||||
|
||||
public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
for (AbstractPageBlock textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
@ -16,21 +16,19 @@ import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.TableServiceResponseAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -46,18 +44,18 @@ public class PdfSegmentationService {
|
||||
private final BlockificationService blockificationService;
|
||||
private final ClassificationService classificationService;
|
||||
private final SectionsBuilderService sectionsBuilderService;
|
||||
private final ImageService imageService;
|
||||
private final TableService tableService;
|
||||
private final ImageServiceResponseAdapter imageServiceResponseAdapter;
|
||||
private final TableServiceResponseAdapter tableServiceResponseAdapter;
|
||||
|
||||
|
||||
public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {
|
||||
public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<ClassifiedImage>> pdfImages) throws IOException {
|
||||
|
||||
PDDocument pdDocument = null;
|
||||
File tempFile = null;
|
||||
try {
|
||||
Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
|
||||
if (redactionServiceSettings.isCvTableParsingEnabled()) {
|
||||
pdfTableCells = tableService.convertTables(dossierId, fileId);
|
||||
pdfTableCells = tableServiceResponseAdapter.convertTables(dossierId, fileId);
|
||||
}
|
||||
|
||||
tempFile = FileUtils.createTempFile("document", ".pdf");
|
||||
@ -94,12 +92,12 @@ public class PdfSegmentationService {
|
||||
}
|
||||
|
||||
|
||||
private void processPage(Map<Integer, List<PdfImage>> pdfImages,
|
||||
PDDocument pdDocument,
|
||||
Map<Integer, List<PdfTableCell>> pdfTableCells,
|
||||
Document document,
|
||||
List<Page> pages,
|
||||
int pageNumber) throws IOException {
|
||||
private void processPage(Map<Integer, List<ClassifiedImage>> pdfImages,
|
||||
PDDocument pdDocument,
|
||||
Map<Integer, List<PdfTableCell>> pdfTableCells,
|
||||
Document document,
|
||||
List<Page> pages,
|
||||
int pageNumber) throws IOException {
|
||||
|
||||
PDFLinesTextStripper stripper = new PDFLinesTextStripper();
|
||||
PDPage pdPage = pdDocument.getPage(pageNumber - 1);
|
||||
@ -130,7 +128,7 @@ public class PdfSegmentationService {
|
||||
// If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
|
||||
if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
|
||||
page.setImages(pdfImages.get(pageNumber));
|
||||
imageService.findOcr(page);
|
||||
imageServiceResponseAdapter.findOcr(page);
|
||||
}
|
||||
|
||||
tableExtractionService.extractTables(cleanRulings, page);
|
||||
@ -155,7 +153,7 @@ public class PdfSegmentationService {
|
||||
private void buildPageStatistics(Page page) {
|
||||
|
||||
// Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
for (AbstractPageBlock textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
if (((TextBlock) textBlock).getSequences() == null) {
|
||||
continue;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.service;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
@ -12,11 +12,11 @@ import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
@ -10,17 +10,17 @@ import java.util.stream.Collectors;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@ -30,13 +30,13 @@ public class SectionsBuilderService {
|
||||
|
||||
public void buildSections(Document document) {
|
||||
|
||||
List<AbstractTextContainer> chunkWords = new ArrayList<>();
|
||||
List<AbstractPageBlock> chunkWords = new ArrayList<>();
|
||||
List<Section> chunkBlockList = new ArrayList<>();
|
||||
List<Header> headers = new ArrayList<>();
|
||||
List<Footer> footers = new ArrayList<>();
|
||||
List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
AbstractPageBlock prev = null;
|
||||
|
||||
String lastHeadline = "";
|
||||
Table previousTable = null;
|
||||
@ -44,7 +44,7 @@ public class SectionsBuilderService {
|
||||
List<TextBlock> header = new ArrayList<>();
|
||||
List<TextBlock> footer = new ArrayList<>();
|
||||
List<TextBlock> unclassifiedText = new ArrayList<>();
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
for (AbstractPageBlock current : page.getTextBlocks()) {
|
||||
|
||||
if (current.getClassification() == null) {
|
||||
continue;
|
||||
@ -114,7 +114,7 @@ public class SectionsBuilderService {
|
||||
|
||||
Map<Integer, List<Section>> sectionMap = new HashMap<>();
|
||||
for (Section section : document.getSections()) {
|
||||
for (AbstractTextContainer container : section.getPageBlocks()) {
|
||||
for (AbstractPageBlock container : section.getPageBlocks()) {
|
||||
|
||||
List<Section> sectionsOnPage = sectionMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>());
|
||||
if (sectionsOnPage.contains(section)) {
|
||||
@ -139,7 +139,7 @@ public class SectionsBuilderService {
|
||||
}
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
for (PdfImage image : page.getImages()) {
|
||||
for (ClassifiedImage image : page.getImages()) {
|
||||
List<Section> sectionsOnPage = sectionMap.get(page.getPageNumber());
|
||||
if (sectionsOnPage == null) {
|
||||
int i = page.getPageNumber();
|
||||
@ -154,7 +154,7 @@ public class SectionsBuilderService {
|
||||
Float xMax = null;
|
||||
Float yMax = null;
|
||||
|
||||
for (AbstractTextContainer abs : section.getPageBlocks()) {
|
||||
for (AbstractPageBlock abs : section.getPageBlocks()) {
|
||||
if (abs.getPage() != page.getPageNumber()) {
|
||||
continue;
|
||||
}
|
||||
@ -240,11 +240,11 @@ public class SectionsBuilderService {
|
||||
}
|
||||
|
||||
|
||||
private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
|
||||
private Section buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
|
||||
|
||||
Section section = new Section();
|
||||
|
||||
for (AbstractTextContainer container : wordBlockList) {
|
||||
for (AbstractPageBlock container : wordBlockList) {
|
||||
if (container instanceof Table table) {
|
||||
|
||||
if (lastHeadline == null || lastHeadline.isEmpty()) {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.service;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
@ -13,15 +13,15 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
|
||||
|
||||
@Service
|
||||
public class TableExtractionService {
|
||||
@ -73,11 +73,11 @@ public class TableExtractionService {
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* <p>
|
||||
* DirAdj (Text direction adjusted) values can not be used here.
|
||||
*
|
||||
* @param cleanRulings The lines used to build the table.
|
||||
* @param page Page object that contains textblocks and statistics.
|
||||
* @param page Page object that contains textblocks and statistics.
|
||||
*/
|
||||
public void extractTables(CleanRulings cleanRulings, Page page) {
|
||||
|
||||
@ -85,8 +85,8 @@ public class TableExtractionService {
|
||||
|
||||
List<TextBlock> toBeRemoved = new ArrayList<>();
|
||||
|
||||
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
|
||||
TextBlock textBlock = (TextBlock) abstractTextContainer;
|
||||
for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
|
||||
TextBlock textBlock = (TextBlock) abstractPageBlock;
|
||||
for (Cell cell : cells) {
|
||||
if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
|
||||
textBlock.getPdfMinY(),
|
||||
@ -119,9 +119,9 @@ public class TableExtractionService {
|
||||
for (Table table : tables) {
|
||||
int position = -1;
|
||||
|
||||
Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
|
||||
Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
|
||||
while (itty.hasNext()) {
|
||||
AbstractTextContainer textBlock = itty.next();
|
||||
AbstractPageBlock textBlock = itty.next();
|
||||
if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) {
|
||||
position = page.getTextBlocks().indexOf(textBlock);
|
||||
}
|
||||
@ -9,7 +9,7 @@
|
||||
* This program is free software under the LGPL (>=v2.1)
|
||||
* Read the file LICENSE.txt coming with the sources for details.
|
||||
*/
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@ -23,9 +23,11 @@ public class FileUtils {
|
||||
/**
|
||||
* Deletes a file; logs a message with the reason if the deletion fails.
|
||||
* This method is null-safe.
|
||||
*
|
||||
* @param file The file to delete. Can be null.
|
||||
*/
|
||||
public void deleteFile(File file) {
|
||||
|
||||
if (file != null) {
|
||||
try {
|
||||
Files.deleteIfExists(file.toPath());
|
||||
@ -1,7 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@ -58,6 +58,7 @@ public final class PositionUtils {
|
||||
|
||||
}
|
||||
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
|
||||
@ -86,6 +87,7 @@ public final class PositionUtils {
|
||||
|
||||
}
|
||||
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Comparator;
|
||||
@ -1,9 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@ -13,7 +13,7 @@ public final class RulingTextDirAdjustUtil {
|
||||
/**
|
||||
* Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox.
|
||||
* This will get the y position of the text, adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
*
|
||||
* <p>
|
||||
* See org.apache.pdfbox.text.TextPosition
|
||||
*/
|
||||
public Line2D.Float convertToDirAdj(Ruling ruling, float dir, float pageWidth, float pageHeight) {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.Comparator;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,23 +1,23 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.mapper;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.mapper;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER;
|
||||
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.FOOTER;
|
||||
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.HEADER;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Arrays;
|
||||
@ -12,27 +12,27 @@ import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.mapper;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.parseRectangle2D;
|
||||
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations.parseRectangle2D;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
|
||||
public class PropertiesMapper {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static java.util.stream.Collectors.groupingBy;
|
||||
@ -17,31 +17,33 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
@Service
|
||||
public class DocumentGraphFactory {
|
||||
@ -74,9 +76,9 @@ public class DocumentGraphFactory {
|
||||
}
|
||||
|
||||
|
||||
private void addSection(SemanticNode parentNode, List<AbstractTextContainer> pageBlocks, List<PdfImage> images, Context context) {
|
||||
private void addSection(SemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, Context context) {
|
||||
|
||||
Map<Integer, List<AbstractTextContainer>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage));
|
||||
Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
|
||||
SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
|
||||
|
||||
context.sections().add(sectionNode);
|
||||
@ -89,30 +91,30 @@ public class DocumentGraphFactory {
|
||||
tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
|
||||
}
|
||||
sectionNode.setTocId(tocId);
|
||||
Set<AbstractTextContainer> alreadyMerged = new HashSet<>();
|
||||
for (AbstractTextContainer abstractTextContainer : pageBlocks) {
|
||||
Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
|
||||
for (AbstractPageBlock abstractPageBlock : pageBlocks) {
|
||||
|
||||
if (alreadyMerged.contains(abstractTextContainer)) {
|
||||
if (alreadyMerged.contains(abstractPageBlock)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (abstractTextContainer instanceof TextBlock) {
|
||||
List<TextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractTextContainer, pageBlocks);
|
||||
if (abstractPageBlock instanceof TextBlock) {
|
||||
List<TextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
|
||||
alreadyMerged.addAll(textBlocks);
|
||||
addParagraphOrHeadline(sectionNode, (TextBlock) abstractTextContainer, context, textBlocks);
|
||||
addParagraphOrHeadline(sectionNode, (TextBlock) abstractPageBlock, context, textBlocks);
|
||||
}
|
||||
if (abstractTextContainer instanceof Table) {
|
||||
addTable(sectionNode, (Table) abstractTextContainer, context);
|
||||
if (abstractPageBlock instanceof Table) {
|
||||
addTable(sectionNode, (Table) abstractPageBlock, context);
|
||||
}
|
||||
}
|
||||
for (PdfImage image : images) {
|
||||
for (ClassifiedImage image : images) {
|
||||
|
||||
addImage(sectionNode, image, context);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
|
||||
private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
|
||||
|
||||
return pageBlocks.stream()
|
||||
.filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
|
||||
@ -173,7 +175,7 @@ public class DocumentGraphFactory {
|
||||
.build();
|
||||
page.getMainBody().add(tableCellNode);
|
||||
|
||||
com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock;
|
||||
com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock textBlock;
|
||||
|
||||
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
|
||||
tableCellNode.setTocId(tocId);
|
||||
@ -188,7 +190,7 @@ public class DocumentGraphFactory {
|
||||
tableCellNode.setTerminal(true);
|
||||
|
||||
} else if (firstTextBlockIsHeadline(cell)) {
|
||||
addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context);
|
||||
addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), Collections.emptyList(), context);
|
||||
tableCellNode.setTerminal(false);
|
||||
|
||||
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
|
||||
@ -254,7 +256,7 @@ public class DocumentGraphFactory {
|
||||
}
|
||||
|
||||
|
||||
private void addImage(SectionNode sectionNode, PdfImage image, Context context) {
|
||||
private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
|
||||
|
||||
Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
|
||||
PageNode page = getPage(image.getPage(), context);
|
||||
@ -279,13 +281,13 @@ public class DocumentGraphFactory {
|
||||
.stream()
|
||||
.map(Header::getTextBlocks)
|
||||
.flatMap(List::stream)
|
||||
.collect(groupingBy(AbstractTextContainer::getPage, toList()));
|
||||
.collect(groupingBy(AbstractPageBlock::getPage, toList()));
|
||||
|
||||
Map<Integer, List<TextBlock>> footers = document.getFooters()
|
||||
.stream()
|
||||
.map(Footer::getTextBlocks)
|
||||
.flatMap(List::stream)
|
||||
.collect(groupingBy(AbstractTextContainer::getPage, toList()));
|
||||
.collect(groupingBy(AbstractPageBlock::getPage, toList()));
|
||||
|
||||
for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
|
||||
if (headers.containsKey(pageIndex)) {
|
||||
@ -383,7 +385,7 @@ public class DocumentGraphFactory {
|
||||
|
||||
|
||||
record Context(
|
||||
TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<PdfImage> images, TextBlockFactory textBlockFactory) {
|
||||
TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<ClassifiedImage> images, TextBlockFactory textBlockFactory) {
|
||||
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
@ -11,10 +11,11 @@ import lombok.experimental.FieldDefaults;
|
||||
@Builder
|
||||
@Getter
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class SearchTextWithTextPositionModel {
|
||||
public class SearchTextWithTextPositionDto {
|
||||
|
||||
String searchText;
|
||||
List<Integer> lineBreaks;
|
||||
List<Integer> stringCoordsToPositionCoords;
|
||||
List<Rectangle2D> positions;
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
@ -7,19 +7,19 @@ import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
|
||||
public class SearchTextWithTextPositionFactory {
|
||||
|
||||
public static final int HEIGHT_PADDING = 2;
|
||||
|
||||
|
||||
public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
|
||||
public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
|
||||
|
||||
if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
|
||||
return SearchTextWithTextPositionModel.builder()
|
||||
return SearchTextWithTextPositionDto.builder()
|
||||
.searchText("")
|
||||
.lineBreaks(Collections.emptyList())
|
||||
.positions(Collections.emptyList())
|
||||
@ -80,7 +80,7 @@ public class SearchTextWithTextPositionFactory {
|
||||
.flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
|
||||
.toList();
|
||||
|
||||
return SearchTextWithTextPositionModel.builder()
|
||||
return SearchTextWithTextPositionDto.builder()
|
||||
.searchText(sb.toString())
|
||||
.lineBreaks(lineBreaksStringIdx)
|
||||
.stringCoordsToPositionCoords(stringIdxToPositionIdx)
|
||||
@ -1,15 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
|
||||
public class TextBlockFactory {
|
||||
|
||||
@ -37,19 +37,19 @@ public class TextBlockFactory {
|
||||
Integer numberOnPage,
|
||||
PageNode page) {
|
||||
|
||||
SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
|
||||
int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
|
||||
SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
|
||||
int offset = stringOffset.getAndAdd(searchTextWithTextPositionDto.getSearchText().length());
|
||||
|
||||
return AtomicTextBlock.builder()
|
||||
.id(textBlockIdx.getAndIncrement())
|
||||
.parent(parent)
|
||||
.searchText(searchTextWithTextPositionModel.getSearchText())
|
||||
.searchText(searchTextWithTextPositionDto.getSearchText())
|
||||
.numberOnPage(numberOnPage)
|
||||
.page(page)
|
||||
.lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
|
||||
.positions(searchTextWithTextPositionModel.getPositions())
|
||||
.stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
|
||||
.boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length()))
|
||||
.lineBreaks(searchTextWithTextPositionDto.getLineBreaks())
|
||||
.positions(searchTextWithTextPositionDto.getPositions())
|
||||
.stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords())
|
||||
.boundary(new Boundary(offset, offset + searchTextWithTextPositionDto.getSearchText().length()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
@ -9,13 +9,13 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -55,7 +55,7 @@ public class DocumentGraph implements SemanticNode {
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> getEntities() {
|
||||
public Set<RedactionEntity> getEntities() {
|
||||
|
||||
return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -9,10 +9,10 @@ import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@ -12,9 +12,9 @@ import java.util.Set;
|
||||
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class EntityNode {
|
||||
public class RedactionEntity {
|
||||
|
||||
// initial values
|
||||
final Boundary boundary;
|
||||
@ -44,7 +44,7 @@ public class EntityNode {
|
||||
boolean dictionaryEntry;
|
||||
boolean dossierDictionaryEntry;
|
||||
Set<Engine> engines;
|
||||
Set<EntityNode> references;
|
||||
Set<RedactionEntity> references;
|
||||
int matchedRule;
|
||||
String redactionReason;
|
||||
String legalBasis;
|
||||
@ -61,9 +61,9 @@ public class EntityNode {
|
||||
SemanticNode deepestFullyContainingNode;
|
||||
|
||||
|
||||
public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
|
||||
public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) {
|
||||
|
||||
return EntityNode.builder()
|
||||
return RedactionEntity.builder()
|
||||
.type(type)
|
||||
.entityType(entityType)
|
||||
.boundary(boundary)
|
||||
@ -125,21 +125,21 @@ public class EntityNode {
|
||||
}
|
||||
|
||||
|
||||
public boolean containedBy(EntityNode entityNode) {
|
||||
public boolean containedBy(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.containedBy(entityNode.getBoundary());
|
||||
return this.boundary.containedBy(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(EntityNode entityNode) {
|
||||
public boolean contains(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.contains(entityNode.getBoundary());
|
||||
return this.boundary.contains(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(EntityNode entityNode) {
|
||||
public boolean intersects(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.intersects(entityNode.getBoundary());
|
||||
return this.boundary.intersects(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
@ -155,9 +155,9 @@ public class EntityNode {
|
||||
}
|
||||
|
||||
|
||||
public void addReference(EntityNode entityNode) {
|
||||
public void addReference(RedactionEntity redactionEntity) {
|
||||
|
||||
references.add(entityNode);
|
||||
references.add(redactionEntity);
|
||||
}
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -34,7 +34,7 @@ public class FooterNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -34,7 +34,7 @@ public class HeaderNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -34,7 +34,7 @@ public class HeadlineNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
@ -8,10 +8,10 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
@ -53,7 +53,7 @@ public class ImageNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
public enum NodeType {
|
||||
DOCUMENT,
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -37,7 +37,7 @@ public class PageNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -32,7 +32,7 @@ public class ParagraphNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,14 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -33,7 +33,7 @@ public class SectionNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -10,12 +10,12 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
|
||||
|
||||
public interface SemanticNode {
|
||||
|
||||
@ -35,7 +35,7 @@ public interface SemanticNode {
|
||||
*
|
||||
* @return Set of all Entities associated with this Node
|
||||
*/
|
||||
Set<EntityNode> getEntities();
|
||||
Set<RedactionEntity> getEntities();
|
||||
|
||||
|
||||
/**
|
||||
@ -204,22 +204,22 @@ public interface SemanticNode {
|
||||
|
||||
|
||||
/**
|
||||
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode.
|
||||
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
|
||||
* It sets the fields accordingly and recursively calls this function on all its children.
|
||||
*
|
||||
* @param entityNode EntityNode, which is being inserted into the graph
|
||||
* @param redactionEntity RedactionEntity, which is being inserted into the graph
|
||||
*/
|
||||
default void addThisToEntityIfIntersects(EntityNode entityNode) {
|
||||
default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) {
|
||||
|
||||
TextBlock textBlock = buildTextBlock();
|
||||
if (textBlock.getBoundary().intersects(entityNode.getBoundary())) {
|
||||
if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
|
||||
|
||||
if (textBlock.containsBoundary(entityNode.getBoundary())) {
|
||||
entityNode.setDeepestFullyContainingNode(this);
|
||||
if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
|
||||
redactionEntity.setDeepestFullyContainingNode(this);
|
||||
}
|
||||
|
||||
entityNode.addIntersectingNode(this);
|
||||
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode));
|
||||
redactionEntity.addIntersectingNode(this);
|
||||
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashMap;
|
||||
@ -8,10 +8,10 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -44,7 +44,7 @@ public class TableCellNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@ -1,14 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -33,7 +33,7 @@ public class TableNode implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
public Stream<TableCellNode> streamTableCells() {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -7,10 +7,10 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -8,8 +8,8 @@ import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Data;
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -10,8 +10,8 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
|
||||
public interface TextBlock extends CharSequence {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
|
||||
|
||||
@ -14,12 +13,13 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.CharSequenceSearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
|
||||
|
||||
@ -34,11 +34,11 @@ public class EntityCreationService {
|
||||
private final EntityEnrichmentService entityEnrichmentService;
|
||||
|
||||
|
||||
public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.buildTextBlock();
|
||||
List<Boundary> startBoundaries = findBoundariesByString(start, textBlock);
|
||||
List<Boundary> stopBoundaries = findBoundariesByString(stop, textBlock);
|
||||
List<Boundary> startBoundaries = CharSequenceSearchUtils.findBoundariesByString(start, textBlock);
|
||||
List<Boundary> stopBoundaries = CharSequenceSearchUtils.findBoundariesByString(stop, textBlock);
|
||||
|
||||
List<Boundary> entityBoundaries = new LinkedList<>();
|
||||
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
|
||||
@ -59,7 +59,7 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
|
||||
.stream()
|
||||
@ -69,7 +69,7 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.buildTextBlock();
|
||||
SearchImplementation searchImplementation = new SearchImplementation(strings, false);
|
||||
@ -82,10 +82,10 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.buildTextBlock();
|
||||
List<Boundary> boundaries = findBoundariesByString(string, textBlock);
|
||||
List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByString(string, textBlock);
|
||||
return boundaries.stream()
|
||||
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
|
||||
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
|
||||
@ -94,14 +94,14 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
|
||||
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
|
||||
public EntityNode bySemanticNode(SemanticNode node, String type, EntityType entityType) {
|
||||
public RedactionEntity bySemanticNode(SemanticNode node, String type, EntityType entityType) {
|
||||
|
||||
Boundary boundary = node.buildTextBlock().getBoundary();
|
||||
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
|
||||
@ -109,15 +109,15 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public EntityNode byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
|
||||
RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);
|
||||
addEntityToGraph(entity, node.getTableOfContents());
|
||||
return entity;
|
||||
}
|
||||
|
||||
|
||||
public EntityNode byEntities(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
|
||||
public RedactionEntity byEntities(List<RedactionEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
|
||||
throw new IllegalArgumentException("Provided entities can not be merged!");
|
||||
@ -126,10 +126,10 @@ public class EntityCreationService {
|
||||
throw new IllegalArgumentException("more than 2 entities are required to merge!");
|
||||
}
|
||||
|
||||
EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType);
|
||||
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction));
|
||||
RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
|
||||
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(RedactionEntity::isRedaction));
|
||||
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
|
||||
EntityNode entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(EntityNode::getMatchedRule)).orElse(entitiesToMerge.get(0));
|
||||
RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
|
||||
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
|
||||
mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
|
||||
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
|
||||
@ -145,7 +145,7 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
|
||||
public void addEntityToGraph(RedactionEntity entity, TableOfContents tableOfContents) {
|
||||
|
||||
tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity);
|
||||
|
||||
@ -158,7 +158,7 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
private static void addToPages(EntityNode entity) {
|
||||
private static void addToPages(RedactionEntity entity) {
|
||||
|
||||
Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary());
|
||||
entity.getPages().addAll(pages);
|
||||
@ -166,22 +166,22 @@ public class EntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
private static void addEntityToNodeEntitySets(EntityNode entity) {
|
||||
private static void addEntityToNodeEntitySets(RedactionEntity entity) {
|
||||
|
||||
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
|
||||
}
|
||||
|
||||
|
||||
private static boolean allEntitiesIntersectAndHaveSameTypes(List<EntityNode> entitiesToMerge) {
|
||||
private static boolean allEntitiesIntersectAndHaveSameTypes(List<RedactionEntity> entitiesToMerge) {
|
||||
|
||||
if (entitiesToMerge.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
EntityNode previousEntity = entitiesToMerge.get(0);
|
||||
for (EntityNode entityNode : entitiesToMerge.subList(1, entitiesToMerge.size())) {
|
||||
boolean typeMatches = entityNode.getType().equals(previousEntity.getType());
|
||||
boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType());
|
||||
boolean intersects = entityNode.intersects(previousEntity);
|
||||
RedactionEntity previousEntity = entitiesToMerge.get(0);
|
||||
for (RedactionEntity redactionEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
|
||||
boolean typeMatches = redactionEntity.getType().equals(previousEntity.getType());
|
||||
boolean entityTypeMatches = redactionEntity.getEntityType().equals(previousEntity.getEntityType());
|
||||
boolean intersects = redactionEntity.intersects(previousEntity);
|
||||
if (!typeMatches || !entityTypeMatches || !intersects) {
|
||||
return false;
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
@ -8,8 +8,8 @@ import java.util.Objects;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -21,7 +21,7 @@ public class EntityEnrichmentService {
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
|
||||
public void enrichEntity(EntityNode entity, TextBlock textBlock) {
|
||||
public void enrichEntity(RedactionEntity entity, TextBlock textBlock) {
|
||||
|
||||
entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
|
||||
entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
|
||||
@ -91,26 +91,26 @@ public class EntityEnrichmentService {
|
||||
}
|
||||
|
||||
|
||||
public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
|
||||
public static void setFields(RedactionEntity redactionEntity, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
|
||||
|
||||
if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
|
||||
entityNode.setMatchedRule(matchedRule);
|
||||
if (redactionEntity.getMatchedRule() == -1 && matchedRule != -1) {
|
||||
redactionEntity.setMatchedRule(matchedRule);
|
||||
}
|
||||
if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
|
||||
entityNode.setRedactionReason(redactionReason);
|
||||
if (redactionEntity.getRedactionReason().equals("") && redactionReason != null) {
|
||||
redactionEntity.setRedactionReason(redactionReason);
|
||||
}
|
||||
if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
|
||||
entityNode.setLegalBasis(legalBasis);
|
||||
if (redactionEntity.getLegalBasis().equals("") && legalBasis != null) {
|
||||
redactionEntity.setLegalBasis(legalBasis);
|
||||
}
|
||||
if (engine != null) {
|
||||
entityNode.addEngine(engine);
|
||||
redactionEntity.addEngine(engine);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
|
||||
public static void setFields(Collection<RedactionEntity> redactionEntities, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
|
||||
|
||||
entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
|
||||
redactionEntities.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
|
||||
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@ -18,7 +18,7 @@ public class ManualRedactionApplicationService {
|
||||
private final EntityCreationService entityCreationService;
|
||||
|
||||
|
||||
public void resizeEntityAndReinsert(EntityNode entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
|
||||
public void resizeEntityAndReinsert(RedactionEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
|
||||
|
||||
RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
|
||||
.stream()
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
|
||||
|
||||
@ -7,8 +7,8 @@ import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
public class CharSequenceSearchUtils {
|
||||
@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@ -17,9 +17,9 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
public class RectangleTransformations {
|
||||
|
||||
@ -29,9 +29,9 @@ public class RectangleTransformations {
|
||||
}
|
||||
|
||||
|
||||
public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractTextContainer> abstractTextContainers) {
|
||||
public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractPageBlock> abstractPageBlocks) {
|
||||
|
||||
return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
|
||||
return abstractPageBlocks.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
|
||||
}
|
||||
|
||||
|
||||
@ -53,9 +53,9 @@ public class RectangleTransformations {
|
||||
}
|
||||
|
||||
|
||||
public static Rectangle2D toRectangle2D(AbstractTextContainer abstractTextContainer) {
|
||||
public static Rectangle2D toRectangle2D(AbstractPageBlock abstractPageBlock) {
|
||||
|
||||
return new Rectangle2D.Float(abstractTextContainer.getMinX(), abstractTextContainer.getMinY(), abstractTextContainer.getWidth(), abstractTextContainer.getHeight());
|
||||
return new Rectangle2D.Float(abstractPageBlock.getMinX(), abstractPageBlock.getMinY(), abstractPageBlock.getWidth(), abstractPageBlock.getHeight());
|
||||
}
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
|
||||
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
|
||||
public class TextPositionOperations {
|
||||
|
||||
@ -12,10 +12,10 @@ import org.springframework.stereotype.Service;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
|
||||
@ -6,7 +6,7 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@ -20,6 +20,7 @@ import lombok.NoArgsConstructor;
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity implements ReasonHolder {
|
||||
|
||||
private String word;
|
||||
private String type;
|
||||
private boolean redaction;
|
||||
@ -29,7 +30,6 @@ public class Entity implements ReasonHolder {
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user