Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009: Document Tree Structure

Browse Source 
* refactored package structure
This commit is contained in:
Kilian Schuettler 2023-04-21 10:54:33 +02:00
parent 45576c858b
commit 135b43fe95
133 changed files with 954 additions and 928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Orientation.java
View File

@ -1,8 +0,0 @@
package com.iqser.red.service.redaction.v1.server.classification.model;
public enum Orientation {
NONE,
LEFT,
RIGHT
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/ImageServiceResponseAdapter.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.segmentation; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -10,11 +10,11 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image.ImageServiceResponse;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -22,26 +22,26 @@ import lombok.SneakyThrows;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class ImageService { public class ImageServiceResponseAdapter {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final RedactionStorageService redactionStorageService; private final RedactionStorageService redactionStorageService;
@SneakyThrows @SneakyThrows
public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId) { public Map<Integer, List<ClassifiedImage>> convertImages(String dossierId, String fileId) {
var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class); ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);
Map<Integer, List<PdfImage>> images = new HashMap<>(); Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
imageServiceResponse.getData().forEach(imageMetadata -> { imageServiceResponse.getData().forEach(imageMetadata -> {
var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification() var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
.getLabel() .getLabel()
.toUpperCase(Locale.ROOT)) : ImageType.OTHER; .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>()) images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), .add(new ClassifiedImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
imageMetadata.getPosition().getY1(), imageMetadata.getPosition().getY1(),
imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getWidth(),
imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())); imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
@ -53,7 +53,7 @@ public class ImageService {
.getLabel() .getLabel()
.toUpperCase(Locale.ROOT)) : ImageType.OTHER; .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>()) images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), .add(new ClassifiedImage(new RedRectangle2D(imageMetadata.getPosition().getX1(),
imageMetadata.getPosition().getY1(), imageMetadata.getPosition().getY1(),
imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getWidth(),
imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())); imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TableService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/TableServiceResponseAdapter.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.segmentation; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
@ -10,9 +10,9 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableCells;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table.TableServiceResponse;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class TableService { public class TableServiceResponseAdapter {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final RedactionStorageService redactionStorageService; private final RedactionStorageService redactionStorageService;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/Classification.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import lombok.Data; import lombok.Data;
@Data @Data

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/FilterGeometry.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/Filters.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/Geometry.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/ImageFormat.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/ImageMetadata.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/ImageServiceResponse.java
View File

@ -1,13 +1,13 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data; import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data @Data
public class ImageServiceResponse { public class ImageServiceResponse {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/ImageSize.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/Position.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.image; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data; import lombok.Data;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/image/Probability.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.image;
import lombok.Data;
@Data
public class Probability {
private boolean unconfident;
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/PageInfo.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/table/PageInfo.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.table; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableCells.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/table/TableCells.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.table; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
import lombok.Data; import lombok.Data;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/table/TableData.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.table; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;
@Data @Data

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableServiceResponse.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/table/TableServiceResponse.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.table; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.table;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;
@Data @Data

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
@ -12,7 +11,7 @@ import lombok.NoArgsConstructor;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
public abstract class AbstractTextContainer { public abstract class AbstractPageBlock {
@JsonIgnore @JsonIgnore
protected float minX; protected float minX;
@ -40,7 +39,7 @@ public abstract class AbstractTextContainer {
} }
public boolean contains(AbstractTextContainer other) { public boolean contains(AbstractPageBlock other) {
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY; return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
} }
@ -67,7 +66,7 @@ public abstract class AbstractTextContainer {
} }
public boolean intersectsY(AbstractTextContainer atc) { public boolean intersectsY(AbstractPageBlock atc) {
return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY(); return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Document.java
View File

@ -1,9 +1,11 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import lombok.Data; import lombok.Data;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/FloatFrequencyCounter.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/FloatFrequencyCounter.java
View File

@ -1,6 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import lombok.Getter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -9,6 +7,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.Getter;
public class FloatFrequencyCounter { public class FloatFrequencyCounter {
@Getter @Getter

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Footer.java
View File

@ -1,13 +1,14 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import java.util.List;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class Footer { public class Footer {

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Header.java
View File

@ -1,13 +1,14 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import java.util.List;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class Header { public class Header {

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Orientation.java Normal file
View File

@ -0,0 +1,8 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
public enum Orientation {
NONE,
LEFT,
RIGHT
}

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Page.java
View File

@ -1,11 +1,11 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
import lombok.Data; import lombok.Data;
import lombok.NonNull; import lombok.NonNull;
@ -16,9 +16,9 @@ import lombok.RequiredArgsConstructor;
public class Page { public class Page {
@NonNull @NonNull
private List<AbstractTextContainer> textBlocks; private List<AbstractPageBlock> textBlocks;
private List<PdfImage> images = new ArrayList<>(); private List<ClassifiedImage> images = new ArrayList<>();
private Rectangle bodyTextFrame; private Rectangle bodyTextFrame;

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Section.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@ -15,8 +15,8 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor @NoArgsConstructor
public class Section implements Comparable { public class Section implements Comparable {
private List<AbstractTextContainer> pageBlocks = new ArrayList<>(); private List<AbstractPageBlock> pageBlocks = new ArrayList<>();
private List<PdfImage> images = new ArrayList<>(); private List<ClassifiedImage> images = new ArrayList<>();
private String headline; private String headline;
@ -61,4 +61,5 @@ public class Section implements Comparable {
return 0; return 0;
} }
} }

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/image/ClassifiedImage.java
View File

@ -1,4 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import lombok.Data; import lombok.Data;
import lombok.NonNull; import lombok.NonNull;
@ -6,7 +9,7 @@ import lombok.RequiredArgsConstructor;
@Data @Data
@RequiredArgsConstructor @RequiredArgsConstructor
public class PdfImage { public class ClassifiedImage {
@NonNull @NonNull
private RedRectangle2D position; private RedRectangle2D position;

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java
View File

@ -1,18 +1,18 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/CellPosition.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/CellPosition.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.Value; import lombok.Value;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/CleanRulings.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/CleanRulings.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import java.util.List;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import java.util.List;
@Data @Data
@Builder @Builder
public class CleanRulings { public class CleanRulings {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Rectangle.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Rectangle.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Ruling.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Ruling.java
View File

@ -1,14 +1,20 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
import lombok.extern.slf4j.Slf4j;
import java.awt.geom.Line2D; import java.awt.geom.Line2D;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Formatter;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.CohenSutherlandClipping;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@SuppressWarnings("all") @SuppressWarnings("all")

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Table.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -12,14 +11,15 @@ import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class Table extends AbstractTextContainer { public class Table extends AbstractPageBlock {
private final TreeMap<CellPosition, Cell> cells = new TreeMap<>(); private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/RedTextPosition.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.parsing.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import org.apache.pdfbox.text.TextPosition; import org.apache.pdfbox.text.TextPosition;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -7,15 +7,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.dslplatform.json.CompiledJson;
import com.dslplatform.json.JsonAttribute;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SimplifiedSectionText.java
View File

@ -1,5 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SimplifiedText.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/StringFrequencyCounter.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/StringFrequencyCounter.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import lombok.Getter;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import lombok.Getter;
public class StringFrequencyCounter { public class StringFrequencyCounter {
@Getter @Getter

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/Text.java
View File

@ -1,13 +1,12 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data @Data
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor @AllArgsConstructor

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextBlock.java
View File

@ -1,13 +1,11 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@ -18,7 +16,7 @@ import lombok.NoArgsConstructor;
@Builder @Builder
@Data @Data
@NoArgsConstructor @NoArgsConstructor
public class TextBlock extends AbstractTextContainer { public class TextBlock extends AbstractPageBlock {
@Builder.Default @Builder.Default
private List<TextPositionSequence> sequences = new ArrayList<>(); private List<TextPositionSequence> sequences = new ArrayList<>();
@ -97,6 +95,7 @@ public class TextBlock extends AbstractTextContainer {
} }
} }
/** /**
* Returns the maxX value in pdf coordinate system. * Returns the maxX value in pdf coordinate system.
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation. * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
@ -177,6 +176,7 @@ public class TextBlock extends AbstractTextContainer {
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) { public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
this.indexOnPage = indexOnPage; this.indexOnPage = indexOnPage;
this.minX = minX; this.minX = minX;
this.maxX = maxX; this.maxX = maxX;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextDirection.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextDirection.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.parsing.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.Objects; import java.util.Objects;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextPositionSequence.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.parsing.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.awt.geom.AffineTransform; import java.awt.geom.AffineTransform;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java
View File

@ -1,4 +1,6 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
@ -6,8 +8,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import java.util.List;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class UnclassifiedText { public class UnclassifiedText {

44
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/LegacyPDFStreamEngine.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/parsing/LegacyPDFStreamEngine.java
View File

@ -14,34 +14,18 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.iqser.red.service.redaction.v1.server.parsing; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
import java.io.InputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.util.Map; import java.util.Map;
import java.util.WeakHashMap; import java.util.WeakHashMap;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.fontbox.ttf.TrueTypeFont; import org.apache.fontbox.ttf.TrueTypeFont;
import org.apache.fontbox.util.BoundingBox; import org.apache.fontbox.util.BoundingBox;
import org.apache.pdfbox.contentstream.PDFStreamEngine; import org.apache.pdfbox.contentstream.PDFStreamEngine;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType3Font;
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
import org.apache.pdfbox.text.TextPosition;
import org.apache.pdfbox.util.Matrix;
import org.apache.pdfbox.util.Vector;
import org.apache.pdfbox.contentstream.operator.DrawObject; import org.apache.pdfbox.contentstream.operator.DrawObject;
import org.apache.pdfbox.contentstream.operator.state.Concatenate; import org.apache.pdfbox.contentstream.operator.state.Concatenate;
import org.apache.pdfbox.contentstream.operator.state.Restore; import org.apache.pdfbox.contentstream.operator.state.Restore;
@ -50,22 +34,36 @@ import org.apache.pdfbox.contentstream.operator.state.SetGraphicsStateParameters
import org.apache.pdfbox.contentstream.operator.state.SetMatrix; import org.apache.pdfbox.contentstream.operator.state.SetMatrix;
import org.apache.pdfbox.contentstream.operator.text.BeginText; import org.apache.pdfbox.contentstream.operator.text.BeginText;
import org.apache.pdfbox.contentstream.operator.text.EndText; import org.apache.pdfbox.contentstream.operator.text.EndText;
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
import org.apache.pdfbox.contentstream.operator.text.MoveText; import org.apache.pdfbox.contentstream.operator.text.MoveText;
import org.apache.pdfbox.contentstream.operator.text.MoveTextSetLeading; import org.apache.pdfbox.contentstream.operator.text.MoveTextSetLeading;
import org.apache.pdfbox.contentstream.operator.text.NextLine; import org.apache.pdfbox.contentstream.operator.text.NextLine;
import org.apache.pdfbox.contentstream.operator.text.SetCharSpacing; import org.apache.pdfbox.contentstream.operator.text.SetCharSpacing;
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
import org.apache.pdfbox.contentstream.operator.text.SetTextLeading; import org.apache.pdfbox.contentstream.operator.text.SetTextLeading;
import org.apache.pdfbox.contentstream.operator.text.SetTextRenderingMode; import org.apache.pdfbox.contentstream.operator.text.SetTextRenderingMode;
import org.apache.pdfbox.contentstream.operator.text.SetTextRise; import org.apache.pdfbox.contentstream.operator.text.SetTextRise;
import org.apache.pdfbox.contentstream.operator.text.SetWordSpacing; import org.apache.pdfbox.contentstream.operator.text.SetWordSpacing;
import org.apache.pdfbox.contentstream.operator.text.ShowText; import org.apache.pdfbox.contentstream.operator.text.ShowText;
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
import org.apache.pdfbox.cos.COSDictionary; import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor; import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType3Font;
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
import org.apache.pdfbox.text.TextPosition;
import org.apache.pdfbox.util.Matrix;
import org.apache.pdfbox.util.Vector;
/** /**
* LEGACY text calculations which are known to be incorrect but are depended on by PDFTextStripper. * LEGACY text calculations which are known to be incorrect but are depended on by PDFTextStripper.

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/parsing/PDFAreaTextStripper.java
View File

@ -1,17 +1,17 @@
package com.iqser.red.service.redaction.v1.server.parsing; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import lombok.Getter;
import lombok.Setter;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import org.apache.pdfbox.text.TextPosition;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import org.apache.pdfbox.text.TextPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import lombok.Getter;
import lombok.Setter;
public class PDFAreaTextStripper extends PDFTextStripperByArea { public class PDFAreaTextStripper extends PDFTextStripperByArea {
@Getter @Getter

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/parsing/PDFLinesTextStripper.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.parsing; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.io.IOException; import java.io.IOException;
@ -34,9 +34,9 @@ import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.text.TextPosition; import org.apache.pdfbox.text.TextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/parsing/PDFTextStripper.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.iqser.red.service.redaction.v1.server.parsing; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.IOException; import java.io.IOException;

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.service; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toSet;
@ -9,15 +9,15 @@ import java.util.List;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation;
import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil;
@Service @Service
@SuppressWarnings("all") @SuppressWarnings("all")
@ -40,7 +40,7 @@ public class BlockificationService {
int indexOnPage = 0; int indexOnPage = 0;
List<TextPositionSequence> chunkWords = new ArrayList<>(); List<TextPositionSequence> chunkWords = new ArrayList<>();
List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>(); List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
float minX = 1000, maxX = 0, minY = 1000, maxY = 0; float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
TextPositionSequence prev = null; TextPositionSequence prev = null;
@ -111,7 +111,7 @@ public class BlockificationService {
chunkBlockList1.add(cb1); chunkBlockList1.add(cb1);
} }
Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator(); Iterator<AbstractPageBlock> itty = chunkBlockList1.iterator();
TextBlock previousLeft = null; TextBlock previousLeft = null;
TextBlock previousRight = null; TextBlock previousRight = null;

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.service; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.util.List; import java.util.List;
@ -6,13 +6,13 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
@Service @Service
public class BodyTextFrameService { public class BodyTextFrameService {
@ -75,7 +75,7 @@ public class BodyTextFrameService {
continue; continue;
} }
for (AbstractTextContainer container : page.getTextBlocks()) { for (AbstractPageBlock container : page.getTextBlocks()) {
if (container instanceof TextBlock) { if (container instanceof TextBlock) {
TextBlock textBlock = (TextBlock) container; TextBlock textBlock = (TextBlock) container;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.classification.service; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -6,11 +6,11 @@ import java.util.regex.Pattern;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -40,7 +40,7 @@ public class ClassificationService {
public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) { public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) {
for (AbstractTextContainer textBlock : page.getTextBlocks()) { for (AbstractPageBlock textBlock : page.getTextBlocks()) {
if (textBlock instanceof TextBlock) { if (textBlock instanceof TextBlock) {
classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes); classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
} }

50
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.segmentation; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -16,21 +16,19 @@ import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle; import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.TableServiceResponseAdapter;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -46,18 +44,18 @@ public class PdfSegmentationService {
private final BlockificationService blockificationService; private final BlockificationService blockificationService;
private final ClassificationService classificationService; private final ClassificationService classificationService;
private final SectionsBuilderService sectionsBuilderService; private final SectionsBuilderService sectionsBuilderService;
private final ImageService imageService; private final ImageServiceResponseAdapter imageServiceResponseAdapter;
private final TableService tableService; private final TableServiceResponseAdapter tableServiceResponseAdapter;
public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException { public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<ClassifiedImage>> pdfImages) throws IOException {
PDDocument pdDocument = null; PDDocument pdDocument = null;
File tempFile = null; File tempFile = null;
try { try {
Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>(); Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
if (redactionServiceSettings.isCvTableParsingEnabled()) { if (redactionServiceSettings.isCvTableParsingEnabled()) {
pdfTableCells = tableService.convertTables(dossierId, fileId); pdfTableCells = tableServiceResponseAdapter.convertTables(dossierId, fileId);
} }
tempFile = FileUtils.createTempFile("document", ".pdf"); tempFile = FileUtils.createTempFile("document", ".pdf");
@ -94,12 +92,12 @@ public class PdfSegmentationService {
} }
private void processPage(Map<Integer, List<PdfImage>> pdfImages, private void processPage(Map<Integer, List<ClassifiedImage>> pdfImages,
PDDocument pdDocument, PDDocument pdDocument,
Map<Integer, List<PdfTableCell>> pdfTableCells, Map<Integer, List<PdfTableCell>> pdfTableCells,
Document document, Document document,
List<Page> pages, List<Page> pages,
int pageNumber) throws IOException { int pageNumber) throws IOException {
PDFLinesTextStripper stripper = new PDFLinesTextStripper(); PDFLinesTextStripper stripper = new PDFLinesTextStripper();
PDPage pdPage = pdDocument.getPage(pageNumber - 1); PDPage pdPage = pdDocument.getPage(pageNumber - 1);
@ -130,7 +128,7 @@ public class PdfSegmentationService {
// If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted. // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
if (pdfImages != null && pdfImages.containsKey(pageNumber)) { if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
page.setImages(pdfImages.get(pageNumber)); page.setImages(pdfImages.get(pageNumber));
imageService.findOcr(page); imageServiceResponseAdapter.findOcr(page);
} }
tableExtractionService.extractTables(cleanRulings, page); tableExtractionService.extractTables(cleanRulings, page);
@ -155,7 +153,7 @@ public class PdfSegmentationService {
private void buildPageStatistics(Page page) { private void buildPageStatistics(Page page) {
// Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame. // Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
for (AbstractTextContainer textBlock : page.getTextBlocks()) { for (AbstractPageBlock textBlock : page.getTextBlocks()) {
if (textBlock instanceof TextBlock) { if (textBlock instanceof TextBlock) {
if (((TextBlock) textBlock).getSequences() == null) { if (((TextBlock) textBlock).getSequences() == null) {
continue; continue;

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/RulingCleaningService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.service; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.awt.geom.Line2D; import java.awt.geom.Line2D;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
@ -12,11 +12,11 @@ import java.util.Map;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.segmentation; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -10,17 +10,17 @@ import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Header; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -30,13 +30,13 @@ public class SectionsBuilderService {
public void buildSections(Document document) { public void buildSections(Document document) {
List<AbstractTextContainer> chunkWords = new ArrayList<>(); List<AbstractPageBlock> chunkWords = new ArrayList<>();
List<Section> chunkBlockList = new ArrayList<>(); List<Section> chunkBlockList = new ArrayList<>();
List<Header> headers = new ArrayList<>(); List<Header> headers = new ArrayList<>();
List<Footer> footers = new ArrayList<>(); List<Footer> footers = new ArrayList<>();
List<UnclassifiedText> unclassifiedTexts = new ArrayList<>(); List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
AbstractTextContainer prev = null; AbstractPageBlock prev = null;
String lastHeadline = ""; String lastHeadline = "";
Table previousTable = null; Table previousTable = null;
@ -44,7 +44,7 @@ public class SectionsBuilderService {
List<TextBlock> header = new ArrayList<>(); List<TextBlock> header = new ArrayList<>();
List<TextBlock> footer = new ArrayList<>(); List<TextBlock> footer = new ArrayList<>();
List<TextBlock> unclassifiedText = new ArrayList<>(); List<TextBlock> unclassifiedText = new ArrayList<>();
for (AbstractTextContainer current : page.getTextBlocks()) { for (AbstractPageBlock current : page.getTextBlocks()) {
if (current.getClassification() == null) { if (current.getClassification() == null) {
continue; continue;
@ -114,7 +114,7 @@ public class SectionsBuilderService {
Map<Integer, List<Section>> sectionMap = new HashMap<>(); Map<Integer, List<Section>> sectionMap = new HashMap<>();
for (Section section : document.getSections()) { for (Section section : document.getSections()) {
for (AbstractTextContainer container : section.getPageBlocks()) { for (AbstractPageBlock container : section.getPageBlocks()) {
List<Section> sectionsOnPage = sectionMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()); List<Section> sectionsOnPage = sectionMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>());
if (sectionsOnPage.contains(section)) { if (sectionsOnPage.contains(section)) {
@ -139,7 +139,7 @@ public class SectionsBuilderService {
} }
for (Page page : document.getPages()) { for (Page page : document.getPages()) {
for (PdfImage image : page.getImages()) { for (ClassifiedImage image : page.getImages()) {
List<Section> sectionsOnPage = sectionMap.get(page.getPageNumber()); List<Section> sectionsOnPage = sectionMap.get(page.getPageNumber());
if (sectionsOnPage == null) { if (sectionsOnPage == null) {
int i = page.getPageNumber(); int i = page.getPageNumber();
@ -154,7 +154,7 @@ public class SectionsBuilderService {
Float xMax = null; Float xMax = null;
Float yMax = null; Float yMax = null;
for (AbstractTextContainer abs : section.getPageBlocks()) { for (AbstractPageBlock abs : section.getPageBlocks()) {
if (abs.getPage() != page.getPageNumber()) { if (abs.getPage() != page.getPageNumber()) {
continue; continue;
} }
@ -240,11 +240,11 @@ public class SectionsBuilderService {
} }
private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) { private Section buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
Section section = new Section(); Section section = new Section();
for (AbstractTextContainer container : wordBlockList) { for (AbstractPageBlock container : wordBlockList) {
if (container instanceof Table table) { if (container instanceof Table table) {
if (lastHeadline == null || lastHeadline.isEmpty()) { if (lastHeadline == null || lastHeadline.isEmpty()) {

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.service; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import java.util.ArrayList; import java.util.ArrayList;
@ -13,15 +13,15 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
@Service @Service
public class TableExtractionService { public class TableExtractionService {
@ -73,11 +73,11 @@ public class TableExtractionService {
* 90 -> UpperLeft * 90 -> UpperLeft
* 180 -> UpperRight * 180 -> UpperRight
* 270 -> LowerRight * 270 -> LowerRight
* * <p>
* DirAdj (Text direction adjusted) values can not be used here. * DirAdj (Text direction adjusted) values can not be used here.
* *
* @param cleanRulings The lines used to build the table. * @param cleanRulings The lines used to build the table.
* @param page Page object that contains textblocks and statistics. * @param page Page object that contains textblocks and statistics.
*/ */
public void extractTables(CleanRulings cleanRulings, Page page) { public void extractTables(CleanRulings cleanRulings, Page page) {
@ -85,8 +85,8 @@ public class TableExtractionService {
List<TextBlock> toBeRemoved = new ArrayList<>(); List<TextBlock> toBeRemoved = new ArrayList<>();
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) { for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
TextBlock textBlock = (TextBlock) abstractTextContainer; TextBlock textBlock = (TextBlock) abstractPageBlock;
for (Cell cell : cells) { for (Cell cell : cells) {
if (cell.intersects(textBlock.getPdfMinX(), if (cell.intersects(textBlock.getPdfMinX(),
textBlock.getPdfMinY(), textBlock.getPdfMinY(),
@ -119,9 +119,9 @@ public class TableExtractionService {
for (Table table : tables) { for (Table table : tables) {
int position = -1; int position = -1;
Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator(); Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
while (itty.hasNext()) { while (itty.hasNext()) {
AbstractTextContainer textBlock = itty.next(); AbstractPageBlock textBlock = itty.next();
if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) { if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) {
position = page.getTextBlocks().indexOf(textBlock); position = page.getTextBlocks().indexOf(textBlock);
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/utils/CohenSutherlandClipping.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/CohenSutherlandClipping.java
View File

@ -9,7 +9,7 @@
* This program is free software under the LGPL (>=v2.1) * This program is free software under the LGPL (>=v2.1)
* Read the file LICENSE.txt coming with the sources for details. * Read the file LICENSE.txt coming with the sources for details.
*/ */
package com.iqser.red.service.redaction.v1.server.tableextraction.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import java.awt.geom.Line2D; import java.awt.geom.Line2D;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/utils/FileUtils.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/FileUtils.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -23,9 +23,11 @@ public class FileUtils {
/** /**
* Deletes a file; logs a message with the reason if the deletion fails. * Deletes a file; logs a message with the reason if the deletion fails.
* This method is null-safe. * This method is null-safe.
*
* @param file The file to delete. Can be null. * @param file The file to delete. Can be null.
*/ */
public void deleteFile(File file) { public void deleteFile(File file) {
if (file != null) { if (file != null) {
try { try {
Files.deleteIfExists(file.toPath()); Files.deleteIfExists(file.toPath());

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.classification.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@ -58,6 +58,7 @@ public final class PositionUtils {
} }
// TODO This currently uses pdf coord system. In the futher this should use java coord system. // TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this. // Note: DirAdj (TextDirection Adjusted) can not be user for this.
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) { public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
@ -86,6 +87,7 @@ public final class PositionUtils {
} }
// TODO This currently uses pdf coord system. In the futher this should use java coord system. // TODO This currently uses pdf coord system. In the futher this should use java coord system.
// Note: DirAdj (TextDirection Adjusted) can not be user for this. // Note: DirAdj (TextDirection Adjusted) can not be user for this.
public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) { public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/utils/QuickSort.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/QuickSort.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.Comparator; import java.util.Comparator;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/RulingTextDirAdjustUtil.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/RulingTextDirAdjustUtil.java
View File

@ -1,9 +1,9 @@
package com.iqser.red.service.redaction.v1.server.classification.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import java.awt.geom.Line2D; import java.awt.geom.Line2D;
import java.awt.geom.Point2D; import java.awt.geom.Point2D;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@ -13,7 +13,7 @@ public final class RulingTextDirAdjustUtil {
/** /**
* Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox. * Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox.
* This will get the y position of the text, adjusted so that 0,0 is upper left and it is adjusted based on the text direction. * This will get the y position of the text, adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
* * <p>
* See org.apache.pdfbox.text.TextPosition * See org.apache.pdfbox.text.TextPosition
*/ */
public Line2D.Float convertToDirAdj(Ruling ruling, float dir, float pageWidth, float pageHeight) { public Line2D.Float convertToDirAdj(Ruling ruling, float dir, float pageWidth, float pageHeight) {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/utils/Utils.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/Utils.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.utils; package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.util.Comparator; import java.util.Comparator;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/TableOfContentsData.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java
View File

@ -1,23 +1,23 @@
package com.iqser.red.service.redaction.v1.server.document.mapper; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;

48
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.mapper; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.FOOTER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.HEADER;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.Arrays; import java.util.Arrays;
@ -12,27 +12,27 @@ import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java
View File

@ -1,14 +1,14 @@
package com.iqser.red.service.redaction.v1.server.document.mapper; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.parseRectangle2D; import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations.parseRectangle2D;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
public class PropertiesMapper { public class PropertiesMapper {

88
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import static java.lang.String.format; import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.groupingBy;
@ -17,31 +17,33 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Header; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@Service @Service
public class DocumentGraphFactory { public class DocumentGraphFactory {
@ -74,9 +76,9 @@ public class DocumentGraphFactory {
} }
private void addSection(SemanticNode parentNode, List<AbstractTextContainer> pageBlocks, List<PdfImage> images, Context context) { private void addSection(SemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, Context context) {
Map<Integer, List<AbstractTextContainer>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage)); Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build(); SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
context.sections().add(sectionNode); context.sections().add(sectionNode);
@ -89,30 +91,30 @@ public class DocumentGraphFactory {
tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode); tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
} }
sectionNode.setTocId(tocId); sectionNode.setTocId(tocId);
Set<AbstractTextContainer> alreadyMerged = new HashSet<>(); Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
for (AbstractTextContainer abstractTextContainer : pageBlocks) { for (AbstractPageBlock abstractPageBlock : pageBlocks) {
if (alreadyMerged.contains(abstractTextContainer)) { if (alreadyMerged.contains(abstractPageBlock)) {
continue; continue;
} }
if (abstractTextContainer instanceof TextBlock) { if (abstractPageBlock instanceof TextBlock) {
List<TextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractTextContainer, pageBlocks); List<TextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
alreadyMerged.addAll(textBlocks); alreadyMerged.addAll(textBlocks);
addParagraphOrHeadline(sectionNode, (TextBlock) abstractTextContainer, context, textBlocks); addParagraphOrHeadline(sectionNode, (TextBlock) abstractPageBlock, context, textBlocks);
} }
if (abstractTextContainer instanceof Table) { if (abstractPageBlock instanceof Table) {
addTable(sectionNode, (Table) abstractTextContainer, context); addTable(sectionNode, (Table) abstractPageBlock, context);
} }
} }
for (PdfImage image : images) { for (ClassifiedImage image : images) {
addImage(sectionNode, image, context); addImage(sectionNode, image, context);
} }
} }
private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) { private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
return pageBlocks.stream() return pageBlocks.stream()
.filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
@ -173,7 +175,7 @@ public class DocumentGraphFactory {
.build(); .build();
page.getMainBody().add(tableCellNode); page.getMainBody().add(tableCellNode);
com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock; com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock textBlock;
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode); List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
tableCellNode.setTocId(tocId); tableCellNode.setTocId(tocId);
@ -188,7 +190,7 @@ public class DocumentGraphFactory {
tableCellNode.setTerminal(true); tableCellNode.setTerminal(true);
} else if (firstTextBlockIsHeadline(cell)) { } else if (firstTextBlockIsHeadline(cell)) {
addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context); addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), Collections.emptyList(), context);
tableCellNode.setTerminal(false); tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
@ -254,7 +256,7 @@ public class DocumentGraphFactory {
} }
private void addImage(SectionNode sectionNode, PdfImage image, Context context) { private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition()); Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
PageNode page = getPage(image.getPage(), context); PageNode page = getPage(image.getPage(), context);
@ -279,13 +281,13 @@ public class DocumentGraphFactory {
.stream() .stream()
.map(Header::getTextBlocks) .map(Header::getTextBlocks)
.flatMap(List::stream) .flatMap(List::stream)
.collect(groupingBy(AbstractTextContainer::getPage, toList())); .collect(groupingBy(AbstractPageBlock::getPage, toList()));
Map<Integer, List<TextBlock>> footers = document.getFooters() Map<Integer, List<TextBlock>> footers = document.getFooters()
.stream() .stream()
.map(Footer::getTextBlocks) .map(Footer::getTextBlocks)
.flatMap(List::stream) .flatMap(List::stream)
.collect(groupingBy(AbstractTextContainer::getPage, toList())); .collect(groupingBy(AbstractPageBlock::getPage, toList()));
for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) { for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
if (headers.containsKey(pageIndex)) { if (headers.containsKey(pageIndex)) {
@ -383,7 +385,7 @@ public class DocumentGraphFactory {
record Context( record Context(
TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<PdfImage> images, TextBlockFactory textBlockFactory) { TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<ClassifiedImage> images, TextBlockFactory textBlockFactory) {
} }

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionModel.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.List; import java.util.List;
@ -11,10 +11,11 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@Getter @Getter
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class SearchTextWithTextPositionModel { public class SearchTextWithTextPositionDto {
String searchText; String searchText;
List<Integer> lineBreaks; List<Integer> lineBreaks;
List<Integer> stringCoordsToPositionCoords; List<Integer> stringCoordsToPositionCoords;
List<Rectangle2D> positions; List<Rectangle2D> positions;
} }

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import java.awt.geom.AffineTransform; import java.awt.geom.AffineTransform;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
@ -7,19 +7,19 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextDirection;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
public class SearchTextWithTextPositionFactory { public class SearchTextWithTextPositionFactory {
public static final int HEIGHT_PADDING = 2; public static final int HEIGHT_PADDING = 2;
public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) { public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) { if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
return SearchTextWithTextPositionModel.builder() return SearchTextWithTextPositionDto.builder()
.searchText("") .searchText("")
.lineBreaks(Collections.emptyList()) .lineBreaks(Collections.emptyList())
.positions(Collections.emptyList()) .positions(Collections.emptyList())
@ -80,7 +80,7 @@ public class SearchTextWithTextPositionFactory {
.flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence))) .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
.toList(); .toList();
return SearchTextWithTextPositionModel.builder() return SearchTextWithTextPositionDto.builder()
.searchText(sb.toString()) .searchText(sb.toString())
.lineBreaks(lineBreaksStringIdx) .lineBreaks(lineBreaksStringIdx)
.stringCoordsToPositionCoords(stringIdxToPositionIdx) .stringCoordsToPositionCoords(stringIdxToPositionIdx)

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java
View File

@ -1,15 +1,15 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
public class TextBlockFactory { public class TextBlockFactory {
@ -37,19 +37,19 @@ public class TextBlockFactory {
Integer numberOnPage, Integer numberOnPage,
PageNode page) { PageNode page) {
SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); int offset = stringOffset.getAndAdd(searchTextWithTextPositionDto.getSearchText().length());
return AtomicTextBlock.builder() return AtomicTextBlock.builder()
.id(textBlockIdx.getAndIncrement()) .id(textBlockIdx.getAndIncrement())
.parent(parent) .parent(parent)
.searchText(searchTextWithTextPositionModel.getSearchText()) .searchText(searchTextWithTextPositionDto.getSearchText())
.numberOnPage(numberOnPage) .numberOnPage(numberOnPage)
.page(page) .page(page)
.lineBreaks(searchTextWithTextPositionModel.getLineBreaks()) .lineBreaks(searchTextWithTextPositionDto.getLineBreaks())
.positions(searchTextWithTextPositionModel.getPositions()) .positions(searchTextWithTextPositionDto.getPositions())
.stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords()) .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords())
.boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length())) .boundary(new Boundary(offset, offset + searchTextWithTextPositionDto.getSearchText().length()))
.build(); .build();
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
import static java.lang.String.format; import static java.lang.String.format;

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentGraph.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.Collections; import java.util.Collections;
@ -9,13 +9,13 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -55,7 +55,7 @@ public class DocumentGraph implements SemanticNode {
} }
public Set<EntityNode> getEntities() { public Set<RedactionEntity> getEntities() {
return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet()); return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
import static java.lang.String.format; import static java.lang.String.format;
@ -9,10 +9,10 @@ import java.util.List;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@ -12,9 +12,9 @@ import java.util.Set;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class EntityNode { public class RedactionEntity {
// initial values // initial values
final Boundary boundary; final Boundary boundary;
@ -44,7 +44,7 @@ public class EntityNode {
boolean dictionaryEntry; boolean dictionaryEntry;
boolean dossierDictionaryEntry; boolean dossierDictionaryEntry;
Set<Engine> engines; Set<Engine> engines;
Set<EntityNode> references; Set<RedactionEntity> references;
int matchedRule; int matchedRule;
String redactionReason; String redactionReason;
String legalBasis; String legalBasis;
@ -61,9 +61,9 @@ public class EntityNode {
SemanticNode deepestFullyContainingNode; SemanticNode deepestFullyContainingNode;
public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) {
return EntityNode.builder() return RedactionEntity.builder()
.type(type) .type(type)
.entityType(entityType) .entityType(entityType)
.boundary(boundary) .boundary(boundary)
@ -125,21 +125,21 @@ public class EntityNode {
} }
public boolean containedBy(EntityNode entityNode) { public boolean containedBy(RedactionEntity redactionEntity) {
return this.boundary.containedBy(entityNode.getBoundary()); return this.boundary.containedBy(redactionEntity.getBoundary());
} }
public boolean contains(EntityNode entityNode) { public boolean contains(RedactionEntity redactionEntity) {
return this.boundary.contains(entityNode.getBoundary()); return this.boundary.contains(redactionEntity.getBoundary());
} }
public boolean intersects(EntityNode entityNode) { public boolean intersects(RedactionEntity redactionEntity) {
return this.boundary.intersects(entityNode.getBoundary()); return this.boundary.intersects(redactionEntity.getBoundary());
} }
@ -155,9 +155,9 @@ public class EntityNode {
} }
public void addReference(EntityNode entityNode) { public void addReference(RedactionEntity redactionEntity) {
references.add(entityNode); references.add(redactionEntity);
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RedactionPosition.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java
View File

@ -1,9 +1,9 @@
package com.iqser.red.service.redaction.v1.server.document.graph.entity; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.List; import java.util.List;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/FooterNode.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -34,7 +34,7 @@ public class FooterNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeaderNode.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -34,7 +34,7 @@ public class HeaderNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -34,7 +34,7 @@ public class HeadlineNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageNode.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.Collections; import java.util.Collections;
@ -8,10 +8,10 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -53,7 +53,7 @@ public class ImageNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
public enum NodeType { public enum NodeType {
DOCUMENT, DOCUMENT,

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/PageNode.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -37,7 +37,7 @@ public class PageNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ParagraphNode.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -32,7 +32,7 @@ public class ParagraphNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java
View File

@ -1,14 +1,14 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -33,7 +33,7 @@ public class SectionNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import static java.lang.String.format; import static java.lang.String.format;
@ -10,12 +10,12 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
public interface SemanticNode { public interface SemanticNode {
@ -35,7 +35,7 @@ public interface SemanticNode {
* *
* @return Set of all Entities associated with this Node * @return Set of all Entities associated with this Node
*/ */
Set<EntityNode> getEntities(); Set<RedactionEntity> getEntities();
/** /**
@ -204,22 +204,22 @@ public interface SemanticNode {
/** /**
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode. * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
* It sets the fields accordingly and recursively calls this function on all its children. * It sets the fields accordingly and recursively calls this function on all its children.
* *
* @param entityNode EntityNode, which is being inserted into the graph * @param redactionEntity RedactionEntity, which is being inserted into the graph
*/ */
default void addThisToEntityIfIntersects(EntityNode entityNode) { default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) {
TextBlock textBlock = buildTextBlock(); TextBlock textBlock = buildTextBlock();
if (textBlock.getBoundary().intersects(entityNode.getBoundary())) { if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
if (textBlock.containsBoundary(entityNode.getBoundary())) { if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
entityNode.setDeepestFullyContainingNode(this); redactionEntity.setDeepestFullyContainingNode(this);
} }
entityNode.addIntersectingNode(this); redactionEntity.addIntersectingNode(this);
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode)); streamChildren().forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
} }
} }

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCellNode.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.HashMap; import java.util.HashMap;
@ -8,10 +8,10 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -44,7 +44,7 @@ public class TableCellNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
@Override @Override

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java
View File

@ -1,14 +1,14 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -33,7 +33,7 @@ public class TableNode implements SemanticNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
Set<EntityNode> entities = new HashSet<>(); Set<RedactionEntity> entities = new HashSet<>();
public Stream<TableCellNode> streamTableCells() { public Stream<TableCellNode> streamTableCells() {

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
import static java.lang.String.format; import static java.lang.String.format;
@ -7,10 +7,10 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
import static java.lang.String.format; import static java.lang.String.format;
@ -8,8 +8,8 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.Data; import lombok.Data;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
import static java.lang.String.format; import static java.lang.String.format;
@ -10,8 +10,8 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
public interface TextBlock extends CharSequence { public interface TextBlock extends CharSequence {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
import java.util.Collections; import java.util.Collections;
import java.util.Set; import java.util.Set;

64
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.redaction.v1.server.document.services; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
@ -14,12 +13,13 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.CharSequenceSearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@ -34,11 +34,11 @@ public class EntityCreationService {
private final EntityEnrichmentService entityEnrichmentService; private final EntityEnrichmentService entityEnrichmentService;
public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { public Set<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock(); TextBlock textBlock = node.buildTextBlock();
List<Boundary> startBoundaries = findBoundariesByString(start, textBlock); List<Boundary> startBoundaries = CharSequenceSearchUtils.findBoundariesByString(start, textBlock);
List<Boundary> stopBoundaries = findBoundariesByString(stop, textBlock); List<Boundary> stopBoundaries = CharSequenceSearchUtils.findBoundariesByString(stop, textBlock);
List<Boundary> entityBoundaries = new LinkedList<>(); List<Boundary> entityBoundaries = new LinkedList<>();
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) { if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
@ -59,7 +59,7 @@ public class EntityCreationService {
} }
public Set<EntityNode> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { public Set<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
.stream() .stream()
@ -69,7 +69,7 @@ public class EntityCreationService {
} }
public Set<EntityNode> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) { public Set<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock(); TextBlock textBlock = node.buildTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, false); SearchImplementation searchImplementation = new SearchImplementation(strings, false);
@ -82,10 +82,10 @@ public class EntityCreationService {
} }
public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) { public Set<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock(); TextBlock textBlock = node.buildTextBlock();
List<Boundary> boundaries = findBoundariesByString(string, textBlock); List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByString(string, textBlock);
return boundaries.stream() return boundaries.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary)) .map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
@ -94,14 +94,14 @@ public class EntityCreationService {
} }
public Set<EntityNode> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { public Set<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock()); List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet()); return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
} }
public EntityNode bySemanticNode(SemanticNode node, String type, EntityType entityType) { public RedactionEntity bySemanticNode(SemanticNode node, String type, EntityType entityType) {
Boundary boundary = node.buildTextBlock().getBoundary(); Boundary boundary = node.buildTextBlock().getBoundary();
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
@ -109,15 +109,15 @@ public class EntityCreationService {
} }
public EntityNode byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);
addEntityToGraph(entity, node.getTableOfContents()); addEntityToGraph(entity, node.getTableOfContents());
return entity; return entity;
} }
public EntityNode byEntities(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) { public RedactionEntity byEntities(List<RedactionEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
throw new IllegalArgumentException("Provided entities can not be merged!"); throw new IllegalArgumentException("Provided entities can not be merged!");
@ -126,10 +126,10 @@ public class EntityCreationService {
throw new IllegalArgumentException("more than 2 entities are required to merge!"); throw new IllegalArgumentException("more than 2 entities are required to merge!");
} }
EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType); RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction)); mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(RedactionEntity::isRedaction));
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
EntityNode entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(EntityNode::getMatchedRule)).orElse(entitiesToMerge.get(0)); RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason()); mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule()); mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis()); mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
@ -145,7 +145,7 @@ public class EntityCreationService {
} }
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { public void addEntityToGraph(RedactionEntity entity, TableOfContents tableOfContents) {
tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity); tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity);
@ -158,7 +158,7 @@ public class EntityCreationService {
} }
private static void addToPages(EntityNode entity) { private static void addToPages(RedactionEntity entity) {
Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary()); Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary());
entity.getPages().addAll(pages); entity.getPages().addAll(pages);
@ -166,22 +166,22 @@ public class EntityCreationService {
} }
private static void addEntityToNodeEntitySets(EntityNode entity) { private static void addEntityToNodeEntitySets(RedactionEntity entity) {
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
} }
private static boolean allEntitiesIntersectAndHaveSameTypes(List<EntityNode> entitiesToMerge) { private static boolean allEntitiesIntersectAndHaveSameTypes(List<RedactionEntity> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) { if (entitiesToMerge.isEmpty()) {
return true; return true;
} }
EntityNode previousEntity = entitiesToMerge.get(0); RedactionEntity previousEntity = entitiesToMerge.get(0);
for (EntityNode entityNode : entitiesToMerge.subList(1, entitiesToMerge.size())) { for (RedactionEntity redactionEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
boolean typeMatches = entityNode.getType().equals(previousEntity.getType()); boolean typeMatches = redactionEntity.getType().equals(previousEntity.getType());
boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType()); boolean entityTypeMatches = redactionEntity.getEntityType().equals(previousEntity.getEntityType());
boolean intersects = entityNode.intersects(previousEntity); boolean intersects = redactionEntity.intersects(previousEntity);
if (!typeMatches || !entityTypeMatches || !intersects) { if (!typeMatches || !entityTypeMatches || !intersects) {
return false; return false;
} }

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.services; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@ -8,8 +8,8 @@ import java.util.Objects;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -21,7 +21,7 @@ public class EntityEnrichmentService {
private final RedactionServiceSettings redactionServiceSettings; private final RedactionServiceSettings redactionServiceSettings;
public void enrichEntity(EntityNode entity, TextBlock textBlock) { public void enrichEntity(RedactionEntity entity, TextBlock textBlock) {
entity.setValue(textBlock.subSequence(entity.getBoundary()).toString()); entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock)); entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
@ -91,26 +91,26 @@ public class EntityEnrichmentService {
} }
public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) { public static void setFields(RedactionEntity redactionEntity, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
if (entityNode.getMatchedRule() == -1 && matchedRule != -1) { if (redactionEntity.getMatchedRule() == -1 && matchedRule != -1) {
entityNode.setMatchedRule(matchedRule); redactionEntity.setMatchedRule(matchedRule);
} }
if (entityNode.getRedactionReason().equals("") && redactionReason != null) { if (redactionEntity.getRedactionReason().equals("") && redactionReason != null) {
entityNode.setRedactionReason(redactionReason); redactionEntity.setRedactionReason(redactionReason);
} }
if (entityNode.getLegalBasis().equals("") && legalBasis != null) { if (redactionEntity.getLegalBasis().equals("") && legalBasis != null) {
entityNode.setLegalBasis(legalBasis); redactionEntity.setLegalBasis(legalBasis);
} }
if (engine != null) { if (engine != null) {
entityNode.addEngine(engine); redactionEntity.addEngine(engine);
} }
} }
public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) { public static void setFields(Collection<RedactionEntity> redactionEntities, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine)); redactionEntities.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
} }
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
View File

@ -1,13 +1,13 @@
package com.iqser.red.service.redaction.v1.server.document.services; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -18,7 +18,7 @@ public class ManualRedactionApplicationService {
private final EntityCreationService entityCreationService; private final EntityCreationService entityCreationService;
public void resizeEntityAndReinsert(EntityNode entityToBeResized, ManualResizeRedaction manualResizeRedaction) { public void resizeEntityAndReinsert(RedactionEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage() RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
.stream() .stream()

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/CharSequenceSearchUtils.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.services; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
@ -7,8 +7,8 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
public class CharSequenceSearchUtils { public class CharSequenceSearchUtils {

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
import static java.lang.String.format; import static java.lang.String.format;
@ -17,9 +17,9 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
public class RectangleTransformations { public class RectangleTransformations {
@ -29,9 +29,9 @@ public class RectangleTransformations {
} }
public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractTextContainer> abstractTextContainers) { public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractPageBlock> abstractPageBlocks) {
return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion()); return abstractPageBlocks.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
} }
@ -53,9 +53,9 @@ public class RectangleTransformations {
} }
public static Rectangle2D toRectangle2D(AbstractTextContainer abstractTextContainer) { public static Rectangle2D toRectangle2D(AbstractPageBlock abstractPageBlock) {
return new Rectangle2D.Float(abstractTextContainer.getMinX(), abstractTextContainer.getMinY(), abstractTextContainer.getWidth(), abstractTextContainer.getHeight()); return new Rectangle2D.Float(abstractPageBlock.getMinX(), abstractPageBlock.getMinY(), abstractPageBlock.getWidth(), abstractPageBlock.getHeight());
} }

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextPositionOperations.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
public class TextPositionOperations { public class TextPositionOperations {

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
View File

@ -12,10 +12,10 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java
View File

@ -1,17 +1,17 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import java.util.ArrayList;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import java.util.Iterator;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@Data @Data
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor @AllArgsConstructor

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java
View File

@ -6,7 +6,7 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@ -20,6 +20,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor @AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true) @EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Entity implements ReasonHolder { public class Entity implements ReasonHolder {
private String word; private String word;
private String type; private String type;
private boolean redaction; private boolean redaction;
@ -29,7 +30,6 @@ public class Entity implements ReasonHolder {
private List<EntityPositionSequence> positionSequences = new ArrayList<>(); private List<EntityPositionSequence> positionSequences = new ArrayList<>();
private List<TextPositionSequence> targetSequences; private List<TextPositionSequence> targetSequences;
@EqualsAndHashCode.Include @EqualsAndHashCode.Include
private Integer start; private Integer start;
@EqualsAndHashCode.Include @EqualsAndHashCode.Include

Some files were not shown because too many files have changed in this diff Show More