From 464f68e573a7215871ffed9831a406acd9a5be0b Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 12 Nov 2024 11:00:06 +0100 Subject: [PATCH] RED-9139: move document to its own module, add TableOfContents and TableOfContentsItem --- .../v1/server/data/old/DocumentPage.java | 25 +++ .../server/data/old/DocumentPositionData.java | 24 +++ .../v1/server/data/old/DocumentStructure.java | 158 ++++++++++++++++++ .../v1/server/data/old/DocumentTextData.java | 28 ++++ .../v1/server/data/old/LayoutEngine.java | 8 + .../v1/server/data/old/NodeType.java | 24 +++ .../storage/DocumentDataFallbackService.java | 8 +- .../management/models/RuleIdentifier.java | 19 +-- 8 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPage.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPositionData.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentStructure.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentTextData.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/LayoutEngine.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/NodeType.java diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPage.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPage.java new file mode 100644 index 00000000..5b46ff27 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPage.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +import java.io.Serializable; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Deprecated +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentPage implements Serializable { + + int number; + int height; + int width; + int rotation; + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPositionData.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPositionData.java new file mode 100644 index 00000000..7e6faf54 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPositionData.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +import java.io.Serializable; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Deprecated +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentPositionData implements Serializable { + + Long id; + int[] stringIdxToPositionIdx; + float[][] positions; + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentStructure.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentStructure.java new file mode 100644 index 00000000..1f3c27a0 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentStructure.java @@ -0,0 +1,158 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +import java.awt.geom.Rectangle2D; +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Deprecated +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentStructure implements Serializable { + + EntryData root; + + public static class TableProperties implements Serializable { + + public static final String NUMBER_OF_ROWS = "numberOfRows"; + public static final String NUMBER_OF_COLS = "numberOfCols"; + + } + + public static class ImageProperties implements Serializable { + + public static final String TRANSPARENT = "transparent"; + public static final String IMAGE_TYPE = "imageType"; + public static final String POSITION = "position"; + public static final String ID = "id"; + + public static final String REPRESENTATION_HASH = "representationHash"; + + } + + public static class TableCellProperties implements Serializable { + + public static final String B_BOX = "bBox"; + public static final String ROW = "row"; + public static final String COL = "col"; + public static final String HEADER = "header"; + + } + + public static class DuplicateParagraphProperties implements Serializable { + + public static final String UNSORTED_TEXTBLOCK_ID = "utbid"; + + } + + public static final String RECTANGLE_DELIMITER = ";"; + + + public static Rectangle2D parseRectangle2D(String bBox) { + + List floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)) + .map(Float::parseFloat) + .toList(); + return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); + } + + + public static double[] parseRepresentationVector(String representationHash) { + + String[] stringArray = representationHash.split("[,\\s]+"); + double[] doubleArray = new double[stringArray.length]; + for (int i = 0; i < stringArray.length; i++) { + doubleArray[i] = Double.parseDouble(stringArray[i]); + } + + return doubleArray; + } + + + public EntryData get(List tocId) { + + if (tocId.isEmpty()) { + return root; + } + EntryData entry = root.children.get(tocId.get(0)); + for (int id : tocId.subList(1, tocId.size())) { + entry = entry.children.get(id); + } + return entry; + } + + + public Stream streamAllEntries() { + + return Stream.concat(Stream.of(root), root.children.stream()) + .flatMap(DocumentStructure::flatten); + } + + + public String toString() { + + return String.join("\n", + streamAllEntries().map(EntryData::toString) + .toList()); + } + + + private static Stream flatten(EntryData entry) { + + return Stream.concat(Stream.of(entry), + entry.children.stream() + .flatMap(DocumentStructure::flatten)); + } + + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + @FieldDefaults(level = AccessLevel.PRIVATE) + public static class EntryData implements Serializable { + + NodeType type; + int[] treeId; + Long[] atomicBlockIds; + Long[] pageNumbers; + Map properties; + List children; + Set engines; + + + @Override + public String toString() { + + StringBuilder sb = new StringBuilder(); + sb.append("["); + for (int i : treeId) { + sb.append(i); + sb.append(","); + } + sb.delete(sb.length() - 1, sb.length()); + sb.append("]: "); + + sb.append(type); + sb.append(" atbs = "); + sb.append(atomicBlockIds.length); + + return sb.toString(); + } + + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentTextData.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentTextData.java new file mode 100644 index 00000000..d8ae3287 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentTextData.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +import java.io.Serializable; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Deprecated +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentTextData implements Serializable { + + Long id; + Long page; + String searchText; + int numberOnPage; + int start; + int end; + int[] lineBreaks; + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/LayoutEngine.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/LayoutEngine.java new file mode 100644 index 00000000..227f01fd --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/LayoutEngine.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +@Deprecated +public enum LayoutEngine { + ALGORITHM, + AI, + OUTLINE +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/NodeType.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/NodeType.java new file mode 100644 index 00000000..fa6c7d53 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/NodeType.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.data.old; + +import java.io.Serializable; +import java.util.Locale; + +@Deprecated +public enum NodeType implements Serializable { + DOCUMENT, + SECTION, + SUPER_SECTION, + HEADLINE, + PARAGRAPH, + TABLE, + TABLE_CELL, + IMAGE, + HEADER, + FOOTER; + + + public String toString() { + + return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/DocumentDataFallbackService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/DocumentDataFallbackService.java index d58b5fe3..0f5d5d2e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/DocumentDataFallbackService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/DocumentDataFallbackService.java @@ -22,11 +22,11 @@ import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto; import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto; import com.iqser.red.service.redaction.v1.server.data.EntryDataProto; import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentPage; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentPositionData; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentStructure; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentTextData; import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java index 1c863af2..569412e4 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java @@ -26,6 +26,15 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { } + public static Set fromListOfIdentifiersString(String input) { + + return Arrays.stream(input.split(",")) + .map(String::trim) + .map(RuleIdentifier::fromString) + .collect(Collectors.toSet()); + } + + public static RuleIdentifier fromString(String identifier) { String[] values = identifier.split("\\."); @@ -36,15 +45,6 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { } - public static Set fromListOfIdentifiersString(String input) { - - return Arrays.stream(input.split(",")) - .map(String::trim) - .map(RuleIdentifier::fromString) - .collect(Collectors.toSet()); - } - - private static Integer parseIntOrStar(String value) { String cleanedValue = value; @@ -65,7 +65,6 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { return ruleIdentifier.type().equals(this.type()) && // (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); - }