Merge branch 'feature/RED-10127-bp' into 'release/0.159.x'

RED-10127: improve headline detection

See merge request fforesight/layout-parser!236

.gitlab-ci.yml

@@ -21,6 +21,5 @@ deploy:
       dotenv: version.env
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
-    - if: $CI_COMMIT_BRANCH =~ /^feature/ && $CI_COMMIT_TAG == ""
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -7,10 +7,4 @@ description = "layoutparser-service-internal-api"
 
 dependencies {
     implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
-    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
-
-    api("com.iqser.red.service:persistence-service-shared-api-v1:2.589.1-RED10196.0") {
-        exclude("org.springframework.boot", "spring-boot-starter-security")
-        exclude("org.springframework.boot", "spring-boot-starter-validation")
-    }
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -2,11 +2,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.io.Serializable;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -18,26 +13,16 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: "
-                      + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. "
-                      + "The text, which is stored as separate blocks of data. "
-                      + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. "
-                      + "The page information.")
+@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: " + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. " + "The text, which is stored as separate blocks of data. " + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. " + "The page information.")
 public class DocumentData implements Serializable {
 
     @Schema(description = "Contains information about the document's pages.")
-    AllDocumentPages documentPages;
+    DocumentPage[] documentPages;
     @Schema(description = "Contains information about the document's text.")
-    AllDocumentTextData documentTextData;
+    DocumentTextData[] documentTextData;
     @Schema(description = "Contains information about the document's text positions.")
-    AllDocumentPositionData documentPositions;
+    DocumentPositionData[] documentPositions;
     @Schema(description = "Contains information about the document's semantic structure.")
-    DocumentStructureWrapper documentStructureWrapper;
-
-
-    public DocumentStructure getDocumentStructure() {
-
-        return documentStructureWrapper.getDocumentStructure();
-    }
+    DocumentStructure documentStructure;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
-@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
-@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -16,7 +16,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
-@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureProto.java

@@ -1,799 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// NO CHECKED-IN PROTOBUF GENCODE
-// source: DocumentStructure.proto
-// Protobuf Java Version: 4.27.1
-@SuppressWarnings("all")
-public final class DocumentStructureProto {
-
-    private DocumentStructureProto() {}
-
-
-    static {
-        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                /* major= */ 4,
-                /* minor= */ 27,
-                /* patch= */ 1,
-                /* suffix= */ "", DocumentStructureProto.class.getName());
-    }
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
-
-    }
-
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-
-        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
-    }
-
-
-    public interface DocumentStructureOrBuilder extends
-            // @@protoc_insertion_point(interface_extends:DocumentStructure)
-            com.google.protobuf.MessageOrBuilder {
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         *
-         * @return Whether the root field is set.
-         */
-        boolean hasRoot();
-
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         *
-         * @return The root.
-         */
-        EntryDataProto.EntryData getRoot();
-
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         */
-        EntryDataProto.EntryDataOrBuilder getRootOrBuilder();
-
-    }
-
-    /**
-     * Protobuf type {@code DocumentStructure}
-     */
-    public static final class DocumentStructure extends com.google.protobuf.GeneratedMessage implements
-            // @@protoc_insertion_point(message_implements:DocumentStructure)
-            DocumentStructureOrBuilder {
-
-        private static final long serialVersionUID = 0L;
-
-        static {
-            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                    /* major= */ 4,
-                    /* minor= */ 27,
-                    /* patch= */ 1,
-                    /* suffix= */ "", DocumentStructure.class.getName());
-        }
-
-        // Use DocumentStructure.newBuilder() to construct.
-        private DocumentStructure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-
-            super(builder);
-        }
-
-
-        private DocumentStructure() {
-
-        }
-
-
-        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
-
-            return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
-        }
-
-
-        @Override
-        protected FieldAccessorTable internalGetFieldAccessorTable() {
-
-            return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
-        }
-
-
-        private int bitField0_;
-        public static final int ROOT_FIELD_NUMBER = 1;
-        private EntryDataProto.EntryData root_;
-
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         *
-         * @return Whether the root field is set.
-         */
-        @Override
-        public boolean hasRoot() {
-
-            return ((bitField0_ & 0x00000001) != 0);
-        }
-
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         *
-         * @return The root.
-         */
-        @Override
-        public EntryDataProto.EntryData getRoot() {
-
-            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
-        }
-
-
-        /**
-         * <pre>
-         * The root EntryData represents the Document.
-         * </pre>
-         *
-         * <code>.EntryData root = 1;</code>
-         */
-        @Override
-        public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
-
-            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
-        }
-
-
-        private byte memoizedIsInitialized = -1;
-
-
-        @Override
-        public final boolean isInitialized() {
-
-            byte isInitialized = memoizedIsInitialized;
-          if (isInitialized == 1) {
-            return true;
-          }
-          if (isInitialized == 0) {
-            return false;
-          }
-
-            memoizedIsInitialized = 1;
-            return true;
-        }
-
-
-        @Override
-        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
-
-            if (((bitField0_ & 0x00000001) != 0)) {
-                output.writeMessage(1, getRoot());
-            }
-            getUnknownFields().writeTo(output);
-        }
-
-
-        @Override
-        public int getSerializedSize() {
-
-            int size = memoizedSize;
-          if (size != -1) {
-            return size;
-          }
-
-            size = 0;
-            if (((bitField0_ & 0x00000001) != 0)) {
-                size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, getRoot());
-            }
-            size += getUnknownFields().getSerializedSize();
-            memoizedSize = size;
-            return size;
-        }
-
-
-        @Override
-        public boolean equals(final Object obj) {
-
-            if (obj == this) {
-                return true;
-            }
-            if (!(obj instanceof DocumentStructure)) {
-                return super.equals(obj);
-            }
-            DocumentStructure other = (DocumentStructure) obj;
-
-          if (hasRoot() != other.hasRoot()) {
-            return false;
-          }
-            if (hasRoot()) {
-              if (!getRoot().equals(other.getRoot())) {
-                return false;
-              }
-            }
-          if (!getUnknownFields().equals(other.getUnknownFields())) {
-            return false;
-          }
-            return true;
-        }
-
-
-        @Override
-        public int hashCode() {
-
-            if (memoizedHashCode != 0) {
-                return memoizedHashCode;
-            }
-            int hash = 41;
-            hash = (19 * hash) + getDescriptor().hashCode();
-            if (hasRoot()) {
-                hash = (37 * hash) + ROOT_FIELD_NUMBER;
-                hash = (53 * hash) + getRoot().hashCode();
-            }
-            hash = (29 * hash) + getUnknownFields().hashCode();
-            memoizedHashCode = hash;
-            return hash;
-        }
-
-
-        public static DocumentStructure parseFrom(java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data);
-        }
-
-
-        public static DocumentStructure parseFrom(java.nio.ByteBuffer data,
-                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data, extensionRegistry);
-        }
-
-
-        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data);
-        }
-
-
-        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data,
-                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data, extensionRegistry);
-        }
-
-
-        public static DocumentStructure parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data);
-        }
-
-
-        public static DocumentStructure parseFrom(byte[] data,
-                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
-
-            return PARSER.parseFrom(data, extensionRegistry);
-        }
-
-
-        public static DocumentStructure parseFrom(java.io.InputStream input) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
-        }
-
-
-        public static DocumentStructure parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
-        }
-
-
-        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input);
-        }
-
-
-        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input, extensionRegistry);
-        }
-
-
-        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
-        }
-
-
-        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input,
-                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
-
-            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
-        }
-
-
-        @Override
-        public Builder newBuilderForType() {return newBuilder();}
-
-
-        public static Builder newBuilder() {
-
-            return DEFAULT_INSTANCE.toBuilder();
-        }
-
-
-        public static Builder newBuilder(DocumentStructure prototype) {
-
-            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-        }
-
-
-        @Override
-        public Builder toBuilder() {
-
-            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
-        }
-
-
-        @Override
-        protected Builder newBuilderForType(BuilderParent parent) {
-
-            Builder builder = new Builder(parent);
-            return builder;
-        }
-
-
-        /**
-         * Protobuf type {@code DocumentStructure}
-         */
-        public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> implements
-                // @@protoc_insertion_point(builder_implements:DocumentStructure)
-                DocumentStructureOrBuilder {
-
-            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
-
-                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
-            }
-
-
-            @Override
-            protected FieldAccessorTable internalGetFieldAccessorTable() {
-
-                return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
-            }
-
-
-            // Construct using DocumentStructureOuterClass.DocumentStructure.newBuilder()
-            private Builder() {
-
-                maybeForceBuilderInitialization();
-            }
-
-
-            private Builder(BuilderParent parent) {
-
-                super(parent);
-                maybeForceBuilderInitialization();
-            }
-
-
-            private void maybeForceBuilderInitialization() {
-
-                if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-                    getRootFieldBuilder();
-                }
-            }
-
-
-            @Override
-            public Builder clear() {
-
-                super.clear();
-                bitField0_ = 0;
-                root_ = null;
-                if (rootBuilder_ != null) {
-                    rootBuilder_.dispose();
-                    rootBuilder_ = null;
-                }
-                return this;
-            }
-
-
-            @Override
-            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
-
-                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
-            }
-
-
-            @Override
-            public DocumentStructure getDefaultInstanceForType() {
-
-                return DocumentStructure.getDefaultInstance();
-            }
-
-
-            @Override
-            public DocumentStructure build() {
-
-                DocumentStructure result = buildPartial();
-                if (!result.isInitialized()) {
-                    throw newUninitializedMessageException(result);
-                }
-                return result;
-            }
-
-
-            @Override
-            public DocumentStructure buildPartial() {
-
-                DocumentStructure result = new DocumentStructure(this);
-                if (bitField0_ != 0) {
-                    buildPartial0(result);
-                }
-                onBuilt();
-                return result;
-            }
-
-
-            private void buildPartial0(DocumentStructure result) {
-
-                int from_bitField0_ = bitField0_;
-                int to_bitField0_ = 0;
-                if (((from_bitField0_ & 0x00000001) != 0)) {
-                    result.root_ = rootBuilder_ == null ? root_ : rootBuilder_.build();
-                    to_bitField0_ |= 0x00000001;
-                }
-                result.bitField0_ |= to_bitField0_;
-            }
-
-
-            @Override
-            public Builder mergeFrom(com.google.protobuf.Message other) {
-
-                if (other instanceof DocumentStructure) {
-                    return mergeFrom((DocumentStructure) other);
-                } else {
-                    super.mergeFrom(other);
-                    return this;
-                }
-            }
-
-
-            public Builder mergeFrom(DocumentStructure other) {
-
-              if (other == DocumentStructure.getDefaultInstance()) {
-                return this;
-              }
-                if (other.hasRoot()) {
-                    mergeRoot(other.getRoot());
-                }
-                this.mergeUnknownFields(other.getUnknownFields());
-                onChanged();
-                return this;
-            }
-
-
-            @Override
-            public final boolean isInitialized() {
-
-                return true;
-            }
-
-
-            @Override
-            public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
-
-                if (extensionRegistry == null) {
-                    throw new NullPointerException();
-                }
-                try {
-                    boolean done = false;
-                    while (!done) {
-                        int tag = input.readTag();
-                        switch (tag) {
-                            case 0:
-                                done = true;
-                                break;
-                            case 10: {
-                                input.readMessage(getRootFieldBuilder().getBuilder(), extensionRegistry);
-                                bitField0_ |= 0x00000001;
-                                break;
-                            } // case 10
-                            default: {
-                                if (!super.parseUnknownField(input, extensionRegistry, tag)) {
-                                    done = true; // was an endgroup tag
-                                }
-                                break;
-                            } // default:
-                        } // switch (tag)
-                    } // while (!done)
-                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-                    throw e.unwrapIOException();
-                } finally {
-                    onChanged();
-                } // finally
-                return this;
-            }
-
-
-            private int bitField0_;
-
-            private EntryDataProto.EntryData root_;
-            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> rootBuilder_;
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             *
-             * @return Whether the root field is set.
-             */
-            public boolean hasRoot() {
-
-                return ((bitField0_ & 0x00000001) != 0);
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             *
-             * @return The root.
-             */
-            public EntryDataProto.EntryData getRoot() {
-
-                if (rootBuilder_ == null) {
-                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
-                } else {
-                    return rootBuilder_.getMessage();
-                }
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public Builder setRoot(EntryDataProto.EntryData value) {
-
-                if (rootBuilder_ == null) {
-                    if (value == null) {
-                        throw new NullPointerException();
-                    }
-                    root_ = value;
-                } else {
-                    rootBuilder_.setMessage(value);
-                }
-                bitField0_ |= 0x00000001;
-                onChanged();
-                return this;
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public Builder setRoot(EntryDataProto.EntryData.Builder builderForValue) {
-
-                if (rootBuilder_ == null) {
-                    root_ = builderForValue.build();
-                } else {
-                    rootBuilder_.setMessage(builderForValue.build());
-                }
-                bitField0_ |= 0x00000001;
-                onChanged();
-                return this;
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public Builder mergeRoot(EntryDataProto.EntryData value) {
-
-                if (rootBuilder_ == null) {
-                    if (((bitField0_ & 0x00000001) != 0) && root_ != null && root_ != EntryDataProto.EntryData.getDefaultInstance()) {
-                        getRootBuilder().mergeFrom(value);
-                    } else {
-                        root_ = value;
-                    }
-                } else {
-                    rootBuilder_.mergeFrom(value);
-                }
-                if (root_ != null) {
-                    bitField0_ |= 0x00000001;
-                    onChanged();
-                }
-                return this;
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public Builder clearRoot() {
-
-                bitField0_ = (bitField0_ & ~0x00000001);
-                root_ = null;
-                if (rootBuilder_ != null) {
-                    rootBuilder_.dispose();
-                    rootBuilder_ = null;
-                }
-                onChanged();
-                return this;
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public EntryDataProto.EntryData.Builder getRootBuilder() {
-
-                bitField0_ |= 0x00000001;
-                onChanged();
-                return getRootFieldBuilder().getBuilder();
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
-
-                if (rootBuilder_ != null) {
-                    return rootBuilder_.getMessageOrBuilder();
-                } else {
-                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
-                }
-            }
-
-
-            /**
-             * <pre>
-             * The root EntryData represents the Document.
-             * </pre>
-             *
-             * <code>.EntryData root = 1;</code>
-             */
-            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> getRootFieldBuilder() {
-
-                if (rootBuilder_ == null) {
-                    rootBuilder_ = new com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder>(getRoot(),
-                                                                                                                                                                             getParentForChildren(),
-                                                                                                                                                                             isClean());
-                    root_ = null;
-                }
-                return rootBuilder_;
-            }
-
-            // @@protoc_insertion_point(builder_scope:DocumentStructure)
-        }
-
-        // @@protoc_insertion_point(class_scope:DocumentStructure)
-        private static final DocumentStructure DEFAULT_INSTANCE;
-
-        static {
-            DEFAULT_INSTANCE = new DocumentStructure();
-        }
-
-        public static DocumentStructure getDefaultInstance() {
-
-            return DEFAULT_INSTANCE;
-        }
-
-
-        private static final com.google.protobuf.Parser<DocumentStructure> PARSER = new com.google.protobuf.AbstractParser<DocumentStructure>() {
-            @Override
-            public DocumentStructure parsePartialFrom(com.google.protobuf.CodedInputStream input,
-                                                      com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
-
-                Builder builder = newBuilder();
-                try {
-                    builder.mergeFrom(input, extensionRegistry);
-                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-                    throw e.setUnfinishedMessage(builder.buildPartial());
-                } catch (com.google.protobuf.UninitializedMessageException e) {
-                    throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial());
-                } catch (java.io.IOException e) {
-                    throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(builder.buildPartial());
-                }
-                return builder.buildPartial();
-            }
-        };
-
-
-        public static com.google.protobuf.Parser<DocumentStructure> parser() {
-
-            return PARSER;
-        }
-
-
-        @Override
-        public com.google.protobuf.Parser<DocumentStructure> getParserForType() {
-
-            return PARSER;
-        }
-
-
-        @Override
-        public DocumentStructure getDefaultInstanceForType() {
-
-            return DEFAULT_INSTANCE;
-        }
-
-    }
-
-    private static final com.google.protobuf.Descriptors.Descriptor internal_static_DocumentStructure_descriptor;
-    private static final com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_DocumentStructure_fieldAccessorTable;
-
-
-    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
-
-        return descriptor;
-    }
-
-
-    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
-
-    static {
-        String[] descriptorData = {"\n\027DocumentStructure.proto\032\017EntryData.pro"
-                                   + "to\"-\n\021DocumentStructure\022\030\n\004root\030\001 \001(\0132\n."
-                                   + "EntryDatab\006proto3"};
-        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData,
-                                                                                                   new com.google.protobuf.Descriptors.FileDescriptor[]{EntryDataProto.getDescriptor(),});
-        internal_static_DocumentStructure_descriptor = getDescriptor().getMessageTypes()
-                .get(0);
-        internal_static_DocumentStructure_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(internal_static_DocumentStructure_descriptor,
-                                                                                                                           new String[]{"Root",});
-        descriptor.resolveAllFeaturesImmutable();
-        EntryDataProto.getDescriptor();
-    }
-
-    // @@protoc_insertion_point(outer_class_scope)
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureWrapper.java

@@ -1,126 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
-
-import java.awt.geom.Rectangle2D;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
-
-import io.swagger.v3.oas.annotations.media.Schema;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-
-@Getter
-@AllArgsConstructor
-public class DocumentStructureWrapper implements Serializable {
-
-    private final DocumentStructure documentStructure;
-
-
-    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
-    public static class TableProperties implements Serializable {
-
-        public static final String NUMBER_OF_ROWS = "numberOfRows";
-        public static final String NUMBER_OF_COLS = "numberOfCols";
-
-    }
-
-    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
-    public static class ImageProperties implements Serializable {
-
-        public static final String TRANSPARENT = "transparent";
-        public static final String IMAGE_TYPE = "imageType";
-        public static final String POSITION = "position";
-        public static final String ID = "id";
-
-        public static final String REPRESENTATION_HASH = "representationHash";
-
-    }
-
-    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
-    public static class TableCellProperties implements Serializable {
-
-        public static final String B_BOX = "bBox";
-        public static final String ROW = "row";
-        public static final String COL = "col";
-        public static final String HEADER = "header";
-
-    }
-
-    @Schema(description = "Object containing the extra field names, a duplicate paragraph has in its properties field.")
-    public static class DuplicateParagraphProperties implements Serializable {
-
-        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
-
-    }
-
-    public static final String RECTANGLE_DELIMITER = ";";
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
-                .map(Float::parseFloat)
-                .toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    public static double[] parseRepresentationVector(String representationHash) {
-
-        String[] stringArray = representationHash.split("[,\\s]+");
-        double[] doubleArray = new double[stringArray.length];
-        for (int i = 0; i < stringArray.length; i++) {
-            doubleArray[i] = Double.parseDouble(stringArray[i]);
-        }
-
-        return doubleArray;
-    }
-
-
-    public EntryData get(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return documentStructure.getRoot();
-        }
-        EntryData entry = documentStructure.getRoot().getChildrenList()
-                .get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.getChildrenList()
-                    .get(id);
-        }
-        return entry;
-    }
-
-
-    public Stream<EntryData> streamAllEntries() {
-
-        return Stream.concat(Stream.of(documentStructure.getRoot()),
-                             documentStructure.getRoot().getChildrenList()
-                                     .stream())
-                .flatMap(DocumentStructureWrapper::flatten);
-    }
-
-
-    public String toString() {
-
-        return String.join("\n",
-                           streamAllEntries().map(EntryData::toString)
-                                   .toList());
-    }
-
-
-    private static Stream<EntryData> flatten(EntryData entry) {
-
-        return Stream.concat(Stream.of(entry),
-                             entry.getChildrenList()
-                                     .stream()
-                                     .flatMap(DocumentStructureWrapper::flatten));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
-@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
-@Deprecated
 public enum LayoutEngine {
     ALGORITHM,
     AI,

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngineProto.java

@@ -1,193 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// NO CHECKED-IN PROTOBUF GENCODE
-// source: LayoutEngine.proto
-// Protobuf Java Version: 4.27.1
-@SuppressWarnings("all")
-public final class LayoutEngineProto {
-
-    private LayoutEngineProto() {}
-
-
-    static {
-        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                /* major= */ 4,
-                /* minor= */ 27,
-                /* patch= */ 1,
-                /* suffix= */ "", LayoutEngineProto.class.getName());
-    }
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
-
-    }
-
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-
-        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
-    }
-
-
-    /**
-     * Protobuf enum {@code LayoutEngine}
-     */
-    public enum LayoutEngine implements com.google.protobuf.ProtocolMessageEnum {
-        /**
-         * <code>ALGORITHM = 0;</code>
-         */
-        ALGORITHM(0),
-        /**
-         * <code>AI = 1;</code>
-         */
-        AI(1),
-        /**
-         * <code>OUTLINE = 2;</code>
-         */
-        OUTLINE(2),
-        UNRECOGNIZED(-1),
-        ;
-
-
-        static {
-            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                    /* major= */ 4,
-                    /* minor= */ 27,
-                    /* patch= */ 1,
-                    /* suffix= */ "", LayoutEngine.class.getName());
-        }
-
-        /**
-         * <code>ALGORITHM = 0;</code>
-         */
-        public static final int ALGORITHM_VALUE = 0;
-        /**
-         * <code>AI = 1;</code>
-         */
-        public static final int AI_VALUE = 1;
-        /**
-         * <code>OUTLINE = 2;</code>
-         */
-        public static final int OUTLINE_VALUE = 2;
-
-
-        public final int getNumber() {
-
-            if (this == UNRECOGNIZED) {
-                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
-            }
-            return value;
-        }
-
-
-        /**
-         * @param value The numeric wire value of the corresponding enum entry.
-         * @return The enum associated with the given numeric wire value.
-         * @deprecated Use {@link #forNumber(int)} instead.
-         */
-        @Deprecated
-        public static LayoutEngine valueOf(int value) {
-
-            return forNumber(value);
-        }
-
-
-        /**
-         * @param value The numeric wire value of the corresponding enum entry.
-         * @return The enum associated with the given numeric wire value.
-         */
-        public static LayoutEngine forNumber(int value) {
-
-            switch (value) {
-                case 0:
-                    return ALGORITHM;
-                case 1:
-                    return AI;
-                case 2:
-                    return OUTLINE;
-                default:
-                    return null;
-            }
-        }
-
-
-        public static com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalGetValueMap() {
-
-            return internalValueMap;
-        }
-
-
-        private static final com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<LayoutEngine>() {
-            public LayoutEngine findValueByNumber(int number) {
-
-                return LayoutEngine.forNumber(number);
-            }
-        };
-
-
-        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
-
-            if (this == UNRECOGNIZED) {
-                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
-            }
-            return getDescriptor().getValues()
-                    .get(ordinal());
-        }
-
-
-        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
-
-            return getDescriptor();
-        }
-
-
-        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
-
-            return LayoutEngineProto.getDescriptor().getEnumTypes()
-                    .get(0);
-        }
-
-
-        private static final LayoutEngine[] VALUES = values();
-
-
-        public static LayoutEngine valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
-
-            if (desc.getType() != getDescriptor()) {
-                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
-            }
-            if (desc.getIndex() == -1) {
-                return UNRECOGNIZED;
-            }
-            return VALUES[desc.getIndex()];
-        }
-
-
-        private final int value;
-
-
-        private LayoutEngine(int value) {
-
-            this.value = value;
-        }
-
-        // @@protoc_insertion_point(enum_scope:LayoutEngine)
-    }
-
-
-    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
-
-        return descriptor;
-    }
-
-
-    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
-
-    static {
-        String[] descriptorData = {"\n\022LayoutEngine.proto*2\n\014LayoutEngine\022\r\n\t" + "ALGORITHM\020\000\022\006\n\002AI\020\001\022\013\n\007OUTLINE\020\002b\006proto3"};
-        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
-        descriptor.resolveAllFeaturesImmutable();
-    }
-
-    // @@protoc_insertion_point(outer_class_scope)
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 import java.io.Serializable;
 import java.util.Locale;
 
-@Deprecated
 public enum NodeType implements Serializable {
     DOCUMENT,
     SECTION,

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeTypeProto.java

@@ -1,274 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
-
-import java.util.Locale;
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// NO CHECKED-IN PROTOBUF GENCODE
-// source: NodeType.proto
-// Protobuf Java Version: 4.27.1
-@SuppressWarnings("all")
-public final class NodeTypeProto {
-
-    private NodeTypeProto() {}
-
-
-    static {
-        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                /* major= */ 4,
-                /* minor= */ 27,
-                /* patch= */ 1,
-                /* suffix= */ "", NodeTypeProto.class.getName());
-    }
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
-
-    }
-
-
-    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-
-        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
-    }
-
-
-    /**
-     * Protobuf enum {@code NodeType}
-     */
-    public enum NodeType implements com.google.protobuf.ProtocolMessageEnum {
-        /**
-         * <code>DOCUMENT = 0;</code>
-         */
-        DOCUMENT(0),
-        /**
-         * <code>SECTION = 1;</code>
-         */
-        SECTION(1),
-        /**
-         * <code>SUPER_SECTION = 2;</code>
-         */
-        SUPER_SECTION(2),
-        /**
-         * <code>HEADLINE = 3;</code>
-         */
-        HEADLINE(3),
-        /**
-         * <code>PARAGRAPH = 4;</code>
-         */
-        PARAGRAPH(4),
-        /**
-         * <code>TABLE = 5;</code>
-         */
-        TABLE(5),
-        /**
-         * <code>TABLE_CELL = 6;</code>
-         */
-        TABLE_CELL(6),
-        /**
-         * <code>IMAGE = 7;</code>
-         */
-        IMAGE(7),
-        /**
-         * <code>HEADER = 8;</code>
-         */
-        HEADER(8),
-        /**
-         * <code>FOOTER = 9;</code>
-         */
-        FOOTER(9),
-        UNRECOGNIZED(-1),
-        ;
-
-
-        public String toString() {
-
-            return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
-        }
-
-
-        static {
-            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
-                    /* major= */ 4,
-                    /* minor= */ 27,
-                    /* patch= */ 1,
-                    /* suffix= */ "", NodeType.class.getName());
-        }
-
-        /**
-         * <code>DOCUMENT = 0;</code>
-         */
-        public static final int DOCUMENT_VALUE = 0;
-        /**
-         * <code>SECTION = 1;</code>
-         */
-        public static final int SECTION_VALUE = 1;
-        /**
-         * <code>SUPER_SECTION = 2;</code>
-         */
-        public static final int SUPER_SECTION_VALUE = 2;
-        /**
-         * <code>HEADLINE = 3;</code>
-         */
-        public static final int HEADLINE_VALUE = 3;
-        /**
-         * <code>PARAGRAPH = 4;</code>
-         */
-        public static final int PARAGRAPH_VALUE = 4;
-        /**
-         * <code>TABLE = 5;</code>
-         */
-        public static final int TABLE_VALUE = 5;
-        /**
-         * <code>TABLE_CELL = 6;</code>
-         */
-        public static final int TABLE_CELL_VALUE = 6;
-        /**
-         * <code>IMAGE = 7;</code>
-         */
-        public static final int IMAGE_VALUE = 7;
-        /**
-         * <code>HEADER = 8;</code>
-         */
-        public static final int HEADER_VALUE = 8;
-        /**
-         * <code>FOOTER = 9;</code>
-         */
-        public static final int FOOTER_VALUE = 9;
-
-
-        public final int getNumber() {
-
-            if (this == UNRECOGNIZED) {
-                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
-            }
-            return value;
-        }
-
-
-        /**
-         * @param value The numeric wire value of the corresponding enum entry.
-         * @return The enum associated with the given numeric wire value.
-         * @deprecated Use {@link #forNumber(int)} instead.
-         */
-        @Deprecated
-        public static NodeType valueOf(int value) {
-
-            return forNumber(value);
-        }
-
-
-        /**
-         * @param value The numeric wire value of the corresponding enum entry.
-         * @return The enum associated with the given numeric wire value.
-         */
-        public static NodeType forNumber(int value) {
-
-            switch (value) {
-                case 0:
-                    return DOCUMENT;
-                case 1:
-                    return SECTION;
-                case 2:
-                    return SUPER_SECTION;
-                case 3:
-                    return HEADLINE;
-                case 4:
-                    return PARAGRAPH;
-                case 5:
-                    return TABLE;
-                case 6:
-                    return TABLE_CELL;
-                case 7:
-                    return IMAGE;
-                case 8:
-                    return HEADER;
-                case 9:
-                    return FOOTER;
-                default:
-                    return null;
-            }
-        }
-
-
-        public static com.google.protobuf.Internal.EnumLiteMap<NodeType> internalGetValueMap() {
-
-            return internalValueMap;
-        }
-
-
-        private static final com.google.protobuf.Internal.EnumLiteMap<NodeType> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<NodeType>() {
-            public NodeType findValueByNumber(int number) {
-
-                return NodeType.forNumber(number);
-            }
-        };
-
-
-        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
-
-            if (this == UNRECOGNIZED) {
-                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
-            }
-            return getDescriptor().getValues()
-                    .get(ordinal());
-        }
-
-
-        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
-
-            return getDescriptor();
-        }
-
-
-        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
-
-            return NodeTypeProto.getDescriptor().getEnumTypes()
-                    .get(0);
-        }
-
-
-        private static final NodeType[] VALUES = values();
-
-
-        public static NodeType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
-
-            if (desc.getType() != getDescriptor()) {
-                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
-            }
-            if (desc.getIndex() == -1) {
-                return UNRECOGNIZED;
-            }
-            return VALUES[desc.getIndex()];
-        }
-
-
-        private final int value;
-
-
-        private NodeType(int value) {
-
-            this.value = value;
-        }
-
-        // @@protoc_insertion_point(enum_scope:NodeType)
-    }
-
-
-    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
-
-        return descriptor;
-    }
-
-
-    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
-
-    static {
-        String[] descriptorData = {"\n\016NodeType.proto*\223\001\n\010NodeType\022\014\n\010DOCUMEN"
-                                   + "T\020\000\022\013\n\007SECTION\020\001\022\021\n\rSUPER_SECTION\020\002\022\014\n\010H"
-                                   + "EADLINE\020\003\022\r\n\tPARAGRAPH\020\004\022\t\n\005TABLE\020\005\022\016\n\nT"
-                                   + "ABLE_CELL\020\006\022\t\n\005IMAGE\020\007\022\n\n\006HEADER\020\010\022\n\n\006FO"
-                                   + "OTER\020\tb\006proto3"};
-        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
-        descriptor.resolveAllFeaturesImmutable();
-    }
-
-    // @@protoc_insertion_point(outer_class_scope)
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -21,14 +21,5 @@ public class SimplifiedText {
     @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
     @Builder.Default
     private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
-    @Schema(description = "A list of the main section numbers ")
-    @Builder.Default
-    private List<String> mainSectionNumbers = new ArrayList<>();
-    @Schema(description = "A list of the header section numbers ")
-    @Builder.Default
-    private List<String> headerSectionNumbers = new ArrayList<>();
-    @Schema(description = "A list of the footer section numbers ")
-    @Builder.Default
-    private List<String> footerSectionNumbers = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,9 +2,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
-    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
-    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
-    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
-    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -3,8 +3,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.NonNull;
@@ -12,7 +10,7 @@ import lombok.NonNull;
 @Builder
 @Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
-        @Schema(description = "Enum specifying the type of layout parsing to be performed.") //
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
         @NonNull LayoutParsingType layoutParsingType,
 
         @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -0,0 +1,12 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.queue;
+
+public enum LayoutParsingType {
+    REDACT_MANAGER,
+    REDACT_MANAGER_OLD,
+    REDACT_MANAGER_PARAGRAPH_DEBUG,
+    REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+    DOCUMINE,
+    DOCUMINE_OLD,
+    CLARIFYND,
+    CLARIFYND_PARAGRAPH_DEBUG
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPage.proto

@@ -1,21 +0,0 @@
-syntax = "proto3";
-
-message AllDocumentPages {
-
-  repeated DocumentPage documentPages = 1;
-}
-
-message DocumentPage {
-  // The page number, starting with 1.
-  int32 number = 1;
-
-  // The page height in PDF user units.
-  int32 height = 2;
-
-  // The page width in PDF user units.
-  int32 width = 3;
-
-  // The page rotation as specified by the PDF.
-  int32 rotation = 4;
-}
-

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPositionData.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-message AllDocumentPositionData {
-
-  repeated DocumentPositionData documentPositionData = 1;
-}
-
-message DocumentPositionData {
-  // Identifier of the text block.
-  int64 id = 1;
-
-  // For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate.
-  // This is required due to the text and position coordinates not being equal.
-  repeated int32 stringIdxToPositionIdx = 2;
-
-  // The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block.
-  // The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner.
-  // In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.
-  repeated Position positions = 3;
-
-  // Definition of a BoundingBox that contains x, y, width, and height.
-  message Position {
-    repeated float value = 1;
-  }
-}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentStructure.proto

@@ -1,8 +0,0 @@
-syntax = "proto3";
-
-import "EntryData.proto";
-
-message DocumentStructure {
-  // The root EntryData represents the Document.
-  EntryData root = 1;
-}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentTextData.proto

@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-message AllDocumentTextData {
-
-  repeated DocumentTextData documentTextData = 1;
-}
-
-message DocumentTextData {
-  // Identifier of the text block.
-  int64 id = 1;
-
-  // The page the text block occurs on.
-  int64 page = 2;
-
-  // The text of the text block.
-  string searchText = 3;
-
-  // Each text block is assigned a number on a page, starting from 0.
-  int32 numberOnPage = 4;
-
-  // The text blocks are ordered, this number represents the start of the text block as a string offset.
-  int32 start = 5;
-
-  // The text blocks are ordered, this number represents the end of the text block as a string offset.
-  int32 end = 6;
-
-  // The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.
-  repeated int32 lineBreaks = 7;
-}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/EntryData.proto

@@ -1,27 +0,0 @@
-syntax = "proto3";
-
-import "LayoutEngine.proto";
-import "NodeType.proto";
-
-message EntryData {
-  // Type of the semantic node.
-  NodeType type = 1;
-
-  // Specifies the position in the parsed tree structure.
-  repeated int32 treeId = 2;
-
-  // Specifies the text block IDs associated with this semantic node.
-  repeated int64 atomicBlockIds = 3;
-
-  // Specifies the pages this semantic node appears on.
-  repeated int64 pageNumbers = 4;
-
-  // Some semantic nodes have additional information, this information is stored in this Map.
-  map<string, string> properties = 5;
-
-  // All child Entries of this Entry.
-  repeated EntryData children = 6;
-
-  // Describes the origin of the semantic node.
-  repeated LayoutEngine engines = 7;
-}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/LayoutEngine.proto

@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-enum LayoutEngine {
-  ALGORITHM = 0;
-  AI = 1;
-  OUTLINE = 2;
-}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/NodeType.proto

@@ -1,14 +0,0 @@
-syntax = "proto3";
-
-enum NodeType {
-  DOCUMENT = 0;
-  SECTION = 1;
-  SUPER_SECTION = 2;
-  HEADLINE = 3;
-  PARAGRAPH = 4;
-  TABLE = 5;
-  TABLE_CELL = 6;
-  IMAGE = 7;
-  HEADER = 8;
-  FOOTER = 9;
-}

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -12,10 +12,12 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
     implementation(project(":viewer-doc-processor"))
 
-    implementation("com.knecon.fforesight:tenant-commons:0.30.0") {
-        exclude("com.iqser.red.commons", "storage-commons")
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
-    implementation("com.iqser.red.commons:storage-commons:2.50.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -32,5 +34,4 @@ dependencies {
     implementation("com.pdftron:PDFNet:10.11.0")
     implementation("org.apache.commons:commons-text:1.12.0")
 
-    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -22,13 +22,14 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.TableOfContentsClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
@@ -44,7 +45,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableO
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
@@ -52,6 +53,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
@@ -61,6 +63,9 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
@@ -125,7 +130,8 @@ public class LayoutParsingPipeline {
         TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId()
                 .map(layoutParsingStorageService::getTablesFile).orElse(new TableServiceResponse());
 
-        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+        ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
+                                                                            ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
                                                                     originFile,
                                                                     imageServiceResponse,
                                                                     tableServiceResponse,
@@ -134,7 +140,8 @@ public class LayoutParsingPipeline {
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
-        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
@@ -268,7 +275,7 @@ public class LayoutParsingPipeline {
             stripper.setEndPage(pageNumber);
             stripper.setPdpage(pdPage);
             stripper.getText(originDocument);
-            List<Word> words = stripper.getWords();
+            List<TextPositionSequence> words = stripper.getTextPositionSequences();
 
 //            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
 
@@ -294,7 +301,7 @@ public class LayoutParsingPipeline {
 
             TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);
 
-            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getTextPositionSequences(), false);
 
             pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
                     .addAll(graphics.stream()
@@ -307,7 +314,7 @@ public class LayoutParsingPipeline {
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
                 case REDACT_MANAGER_OLD ->
-                        redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                 case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                 case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                         docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -367,7 +374,14 @@ public class LayoutParsingPipeline {
 
         classificationService.classify(classificationDocument, layoutParsingType, identifier);
 
-        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
+        List<TextPageBlock> headlines = classificationDocument.getPages()
+                .stream()
+                .flatMap(classificationPage -> classificationPage.getTextBlocks()
+                        .stream()
+                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
+                        .map(tb -> (TextPageBlock) tb))
+                .toList();
+        TableOfContents tableOfContents = outlineValidationService.createToC(headlines);
         classificationDocument.setTableOfContents(tableOfContents);
 
         log.info("Building Sections for {}", identifier);
@@ -381,12 +395,12 @@ public class LayoutParsingPipeline {
     }
 
 
-    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
+    private static void rotateDirAdjExactly(List<TextPositionSequence> words, PDPage pdPage) {
 
         for (TextDirection dir : TextDirection.values()) {
 
             double averageRotation = words.stream()
-                    .map(Word::getTextPositions)
+                    .map(TextPositionSequence::getTextPositions)
                     .flatMap(Collection::stream)
                     .filter(pos -> pos.getDir().equals(dir))
                     .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
@@ -397,7 +411,7 @@ public class LayoutParsingPipeline {
 
             AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
 
-            for (Word word : words) {
+            for (TextPositionSequence word : words) {
                 if (!dir.equals(word.getDir())) {
                     continue;
                 }
@@ -448,10 +462,10 @@ public class LayoutParsingPipeline {
         // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
         for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getWords() == null) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
                     continue;
                 }
-                for (Word word : ((TextPageBlock) textBlock).getWords()) {
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
                     classificationPage.getTextHeightCounter().add(word.getTextHeight());
                     classificationPage.getFontCounter().add(word.getFont());
                     classificationPage.getFontSizeCounter().add(word.getFontSize());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -11,9 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 
-import org.springframework.core.task.TaskExecutor;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -41,8 +39,6 @@ public class LayoutParsingStorageService {
     private final StorageService storageService;
     private final ObjectMapper objectMapper;
 
-    private final TaskExecutor taskExecutor;
-
 
     @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
     public File getOriginFile(String storageId) throws IOException {
@@ -104,35 +100,13 @@ public class LayoutParsingStorageService {
     }
 
 
-    @SneakyThrows
     @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
     public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
-        Runnable storeDocumentStructureRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
-                                                                                       layoutParsingRequest.structureFileStorageId(),
-                                                                                       documentData.getDocumentStructure());
-
-        CompletableFuture<Void> storeDocumentStructureFuture = CompletableFuture.runAsync(storeDocumentStructureRunnable, taskExecutor);
-
-        Runnable storeDocumentTextDataRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
-                                                                                      layoutParsingRequest.textBlockFileStorageId(),
-                                                                                      documentData.getDocumentTextData());
-
-        CompletableFuture<Void> storeDocumentTextDataFuture = CompletableFuture.runAsync(storeDocumentTextDataRunnable, taskExecutor);
-
-        Runnable storeDocumentPositionsRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
-                                                                                       layoutParsingRequest.positionBlockFileStorageId(),
-                                                                                       documentData.getDocumentPositions());
-
-        CompletableFuture<Void> storeDocumentPositionsFuture = CompletableFuture.runAsync(storeDocumentPositionsRunnable, taskExecutor);
-
-        Runnable storeDocumentPagesRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
-                                                                                   layoutParsingRequest.pageFileStorageId(),
-                                                                                   documentData.getDocumentPages());
-
-        CompletableFuture<Void> storeDocumentPagesFuture = CompletableFuture.runAsync(storeDocumentPagesRunnable, taskExecutor);
-
-        CompletableFuture.allOf(storeDocumentStructureFuture, storeDocumentTextDataFuture, storeDocumentPositionsFuture, storeDocumentPagesFuture).join();
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutparserSettings.java

@@ -3,6 +3,8 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+
 import lombok.AccessLevel;
 import lombok.Data;
 import lombok.experimental.FieldDefaults;
@@ -14,5 +16,6 @@ import lombok.experimental.FieldDefaults;
 public class LayoutparserSettings {
 
     boolean debug;
+    LayoutParsingType layoutParsingTypeOverride;
     String pdftronLicense;
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -18,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.Zon
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 import lombok.RequiredArgsConstructor;
@@ -35,7 +35,7 @@ public class DocstrumSegmentationService {
     private final ReadingOrderService readingOrderService;
 
 
-    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
 
         EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);
 
@@ -78,11 +78,11 @@ public class DocstrumSegmentationService {
     }
 
 
-    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+    private List<Zone> computeZones(List<TextPositionSequence> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
 
         List<RedTextPosition> positions = textPositions.stream()
                 .filter(t -> t.getDir() == direction)
-                .map(Word::getTextPositions)
+                .map(TextPositionSequence::getTextPositions)
                 .flatMap(List::stream)
                 .toList();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -1,9 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.STANDARD;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -14,7 +14,7 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
 
 import lombok.Data;
@@ -41,7 +41,7 @@ public class Line extends TextBoundingBox {
     private FontStyle fontStyle;
 
     private final List<Character> characters;
-    private final List<Word> words = new ArrayList<>();
+    private final List<TextPositionSequence> words = new ArrayList<>();
 
 
     public Line(List<Character> characters, double wordSpacing) {
@@ -89,7 +89,7 @@ public class Line extends TextBoundingBox {
         for (FontStyle fontStyle : FontStyle.values()) {
             fontStyleCounter.put(fontStyle, new AtomicInteger(0));
         }
-        for (Word word : words) {
+        for (TextPositionSequence word : words) {
             switch (word.getFontStyle()) {
                 case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
                 case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
@@ -159,14 +159,14 @@ public class Line extends TextBoundingBox {
 
     private void computeWords(double wordSpacing) {
 
-        Word word = new Word();
+        TextPositionSequence word = new TextPositionSequence();
         Character previous = null;
         for (Character current : characters) {
             if (previous != null) {
                 double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
                 if (dist > wordSpacing) {
                     words.add(word);
-                    word = new Word();
+                    word = new TextPositionSequence();
                 }
             }
             word.getTextPositions().add(current.getTextPosition());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/TextBoundingBox.java

@@ -165,16 +165,4 @@ public abstract class TextBoundingBox extends BoundingBox {
         return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
     }
 
-
-    public boolean isAboveDirAdj(TextBoundingBox other) {
-
-        return other.isBelow(this);
-    }
-
-
-    public boolean isBelowDirAdj(TextBoundingBox other) {
-
-        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -4,7 +4,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -51,7 +51,7 @@ public class FloatFrequencyCounter {
                     mostPopular = entry;
                 }
             }
-            mostPopularCache = mostPopular != null ? mostPopular.getKey() : 0;
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
             changed = false;
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/LineInformation.java

@@ -3,7 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 public class LineInformation {
 
     List<Rectangle2D> lineBBox;
-    List<List<Word>> sequencesByLines;
+    List<List<TextPositionSequence>> sequencesByLines;
     List<List<Rectangle2D>> bBoxWithGapsByLines;
-    List<List<List<Word>>> sequencesWithGapsByLines;
+    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -9,13 +9,13 @@ public enum PageBlockType {
     H6,
     HEADER,
     FOOTER,
+    TITLE,
     PARAGRAPH,
     PARAGRAPH_BOLD,
     PARAGRAPH_ITALIC,
     PARAGRAPH_UNKNOWN,
     OTHER,
     TABLE_OF_CONTENTS_ITEM,
-    LIST_ITEM,
     TABLE;
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageContents.java

@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -15,7 +15,7 @@ import lombok.Getter;
 @AllArgsConstructor
 public class PageContents {
 
-    List<Word> sortedWords;
+    List<TextPositionSequence> sortedTextPositionSequences;
     Rectangle2D cropBox;
     Rectangle2D mediaBox;
     List<Ruling> rulings;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/DocumentTree.java

@@ -8,7 +8,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/AbstractSemanticNode.java

@@ -6,7 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -9,7 +9,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -6,7 +6,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java

@@ -12,8 +12,8 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SuperSection.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java

@@ -12,8 +12,8 @@ import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java

@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/AtomicTextBlock.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
-import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
@@ -14,8 +13,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
@@ -154,9 +153,11 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(documentTextData.getStart(), documentTextData.getEnd()))
                 .searchText(documentTextData.getSearchText())
-                .lineBreaks(documentTextData.getLineBreaksList())
-                .stringIdxToPositionIdx(documentPositionData.getStringIdxToPositionIdxList())
-                .positions(toRectangle2DList(documentPositionData.getPositionsList()))
+                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed()
+                                    .toList())
+                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed()
+                                                .toList())
+                .positions(toRectangle2DList(documentPositionData.getPositions()))
                 .parent(parent)
                 .build();
     }
@@ -170,14 +171,6 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {
-
-        return positions.stream()
-                .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getValue(0), pos.getValue(1), pos.getValue(2), pos.getValue(3)))
-                .toList();
-    }
-
-
     public CharSequence getLine(int lineNumber) {
 
         if (lineNumber >= numberOfLines() || lineNumber < 0) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineValidationService.java

@@ -10,7 +10,6 @@ import java.util.TreeSet;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import io.micrometer.observation.annotation.Observed;
@@ -21,9 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 public class OutlineValidationService {
 
     @Observed(name = "OutlineValidationService", contextualName = "create-toc")
-    public TableOfContents createToC(ClassificationDocument classificationDocument) {
-
-        List<TextPageBlock> headlines = extractHeadlines(classificationDocument);
+    public TableOfContents createToC(List<TextPageBlock> headlines) {
 
         List<TableOfContentItem> mainSections = new ArrayList<>();
         Map<Integer, TableOfContentItem> lastItemsPerDepth = new HashMap<>();
@@ -63,16 +60,4 @@ public class OutlineValidationService {
         return new TableOfContents(mainSections);
     }
 
-
-    private static List<TextPageBlock> extractHeadlines(ClassificationDocument classificationDocument) {
-
-        return classificationDocument.getPages()
-                .stream()
-                .flatMap(classificationPage -> classificationPage.getTextBlocks()
-                        .stream()
-                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
-                        .map(tb -> (TextPageBlock) tb))
-                .toList();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TOCEnrichmentService.java

@@ -168,7 +168,7 @@ public class TOCEnrichmentService {
             }
         }
 
-        if (!startBlocks.isEmpty() || !startImages.isEmpty()) {
+        if (!startBlocks.isEmpty()) {
             TableOfContentItem unassigned = new TableOfContentItem(null);
             unassigned.setSectionBlocks(startBlocks);
             unassigned.setImages(startImages);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -9,7 +9,7 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Data;
@@ -68,12 +68,12 @@ public class Cell extends BoundingBox {
         StringBuilder sb = new StringBuilder();
 
         Iterator<TextPageBlock> itty = textBlocks.iterator();
-        Word previous = null;
+        TextPositionSequence previous = null;
         while (itty.hasNext()) {
 
             TextPageBlock textBlock = itty.next();
 
-            for (Word word : textBlock.getWords()) {
+            for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {
                     if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/AbstractBlockOnPage.java

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-
-public record AbstractBlockOnPage(AbstractPageBlock block, ClassificationPage page) {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/ListIdentifier.java

@@ -1,90 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ListIdentifier {
-
-    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]+)\\.\\s+");
-    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]+)\\)\\s+");
-
-    enum Format {
-        NUMBER_WITH_DOT, NUMBER_IN_PARENTHESES
-    }
-
-    Format format;
-    @Getter
-    Word word;
-    @Getter
-    int page;
-    int representation;
-
-
-    public static Optional<ListIdentifier> parse(TextPageBlock textPageBlock, int page) {
-
-        return parse(textPageBlock.getWords().subList(0, Math.min(5, textPageBlock.getWords().size())), page);
-    }
-
-
-    public static Optional<ListIdentifier> parse(List<Word> sequences, int page) {
-
-        StringBuilder sb = new StringBuilder();
-        for (Word sequence : sequences) {
-            sb.append(sequence.toString());
-            sb.append(" ");
-        }
-        sb.replace(sb.length() - 1, sb.length(), "");
-        String text = sb.toString();
-
-        Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
-
-        if (numberMatcher.find()) {
-            return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, Integer.parseInt(numberMatcher.group(1))));
-        }
-
-        Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
-        if (parenthesisMatcher.find()) {
-            return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, Integer.parseInt(parenthesisMatcher.group(1))));
-        }
-        return Optional.empty();
-    }
-
-
-    public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
-
-        if (listIdentifiers.size() <= 1) {
-            return true;
-        }
-
-        for (int i = 1; i < listIdentifiers.size(); i++) {
-            ListIdentifier current = listIdentifiers.get(i);
-            ListIdentifier previous = listIdentifiers.get(i - 1);
-            if (current.format != previous.format) {
-                return false;
-            }
-            if (current.representation <= previous.representation) {
-                return false;
-            }
-            if (!current.word.intersectsXDirAdj(previous.word, 2)) {
-                return false;
-            }
-            if (current.page == previous.page && !current.word.isBelowDirAdj(previous.word)) {
-                return false;
-            }
-            if (current.page < previous.page) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -10,18 +10,18 @@ import lombok.Getter;
 @Getter
 public class SearchableText {
 
-    private final List<Word> sequences = new ArrayList<>();
+    private final List<TextPositionSequence> sequences = new ArrayList<>();
 
 
-    public void add(Word word) {
+    public void add(TextPositionSequence textPositionSequence) {
 
-        sequences.add(word);
+        sequences.add(textPositionSequence);
     }
 
 
-    public void addAll(List<Word> words) {
+    public void addAll(List<TextPositionSequence> textPositionSequences) {
 
-        sequences.addAll(words);
+        sequences.addAll(textPositionSequences);
     }
 
 
@@ -32,10 +32,10 @@ public class SearchableText {
     }
 
 
-    public static String buildString(List<Word> sequences) {
+    public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
-        for (Word word : sequences) {
+        for (TextPositionSequence word : sequences) {
             sb.append(word);
             sb.append(' ');
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextBlockOnPage.java

@@ -2,6 +2,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 
-public record TextBlockOnPage(TextPageBlock textBlock, ClassificationPage page) {
+public record TextBlockOnPage(ClassificationPage page, TextPageBlock textBlock) {
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -25,7 +25,7 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {
 
     @Builder.Default
-    private List<Word> words = new ArrayList<>();
+    private List<TextPositionSequence> sequences = new ArrayList<>();
     @Builder.Default
     private FrequencyCounters frequencyCounters = new FrequencyCounters();
 
@@ -33,6 +33,8 @@ public class TextPageBlock extends AbstractPageBlock {
 
     private boolean underlined;
 
+    private double highestFontSize;
+
     private PageBlockType classification;
 
     private boolean toDuplicate;
@@ -41,43 +43,43 @@ public class TextPageBlock extends AbstractPageBlock {
     private boolean changed;
 
 
-    public TextPageBlock(List<Word> words) {
+    public TextPageBlock(List<TextPositionSequence> sequences) {
 
-        this.words = new ArrayList<>(words);
+        this.sequences = new ArrayList<>(sequences);
         this.frequencyCounters = new FrequencyCounters();
 
-        if (!words.isEmpty()) {
-            addToFrequencyCounters(words);
+        if (!sequences.isEmpty()) {
+            addToFrequencyCounters(sequences);
         }
         calculateBBox();
     }
 
 
-    public List<Word> getWords() {
+    public List<TextPositionSequence> getSequences() {
 
-        return Collections.unmodifiableList(words);
+        return Collections.unmodifiableList(sequences);
     }
 
 
     public TextDirection getDir() {
 
-        return words.get(0).getDir();
+        return sequences.get(0).getDir();
     }
 
 
     private void calculateBBox() {
 
-        if (words == null) {
+        if (sequences == null) {
             this.bBox = new Rectangle2D.Double();
             this.bBoxPdf = new Rectangle2D.Double();
             this.bBoxDirAdj = new Rectangle2D.Double();
             return;
         }
-        this.bBoxDirAdj = words.stream()
-                .map(Word::getBBoxDirAdj)
+        this.bBoxDirAdj = sequences.stream()
+                .map(TextPositionSequence::getBBoxDirAdj)
                 .collect(RectangleTransformations.collectBBox());
 
-        setToBBoxOfComponents(words);
+        setToBBoxOfComponents(sequences);
     }
 
 
@@ -99,8 +101,8 @@ public class TextPageBlock extends AbstractPageBlock {
             throw new IllegalArgumentException("Cannot merge textBlocks on different pages.");
         }
 
-        List<Word> sequences = textBlocksToMerge.stream()
-                .map(TextPageBlock::getWords)
+        List<TextPositionSequence> sequences = textBlocksToMerge.stream()
+                .map(TextPageBlock::getSequences)
                 .flatMap(java.util.Collection::stream)
                 .toList();
         sequences = new ArrayList<>(sequences);
@@ -109,9 +111,9 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-    private void addToFrequencyCounters(List<Word> sequences) {
+    private void addToFrequencyCounters(List<TextPositionSequence> sequences) {
 
-        for (Word wordBlock : sequences) {
+        for (TextPositionSequence wordBlock : sequences) {
 
             frequencyCounters.getLineHeightFrequencyCounter().add(wordBlock.getTextHeight());
             frequencyCounters.getFontSizeFrequencyCounter().add(wordBlock.getFontSize());
@@ -120,12 +122,12 @@ public class TextPageBlock extends AbstractPageBlock {
             frequencyCounters.getStyleFrequencyCounter().add(wordBlock.getFontStyle());
         }
 
-        setUnderlined(this.words.stream()
-                              .allMatch(Word::isUnderline));
+        setUnderlined(this.sequences.stream()
+                              .allMatch(TextPositionSequence::isUnderline));
     }
 
 
-    public TextPageBlock union(Word r) {
+    public TextPageBlock union(TextPositionSequence r) {
 
         TextPageBlock union = this.copy();
         union.add(r);
@@ -138,8 +140,8 @@ public class TextPageBlock extends AbstractPageBlock {
     public TextPageBlock union(TextPageBlock r) {
 
         TextPageBlock union = this.copy();
-        union.addAll(r.getWords());
-        addToFrequencyCounters(r.getWords());
+        union.addAll(r.getSequences());
+        addToFrequencyCounters(r.getSequences());
         calculateBBox();
         return union;
     }
@@ -148,33 +150,33 @@ public class TextPageBlock extends AbstractPageBlock {
     public void add(TextPageBlock textPageBlock) {
 
         changed = true;
-        words.addAll(textPageBlock.getWords());
-        addToFrequencyCounters(textPageBlock.getWords());
+        sequences.addAll(textPageBlock.getSequences());
+        addToFrequencyCounters(textPageBlock.getSequences());
         calculateBBox();
     }
 
 
-    public void add(Word word) {
+    public void add(TextPositionSequence textPositionSequence) {
 
         changed = true;
-        words.add(word);
-        addToFrequencyCounters(List.of(word));
+        sequences.add(textPositionSequence);
+        addToFrequencyCounters(List.of(textPositionSequence));
         calculateBBox();
     }
 
 
-    public void addAll(List<Word> words) {
+    public void addAll(List<TextPositionSequence> textPositionSequences) {
 
         changed = true;
-        this.words.addAll(words);
-        addToFrequencyCounters(words);
+        sequences.addAll(textPositionSequences);
+        addToFrequencyCounters(textPositionSequences);
         calculateBBox();
     }
 
 
     public TextPageBlock copy() {
 
-        return new TextPageBlock(new ArrayList<>(words));
+        return new TextPageBlock(new ArrayList<>(sequences));
     }
 
 
@@ -193,8 +195,8 @@ public class TextPageBlock extends AbstractPageBlock {
 
             StringBuilder sb = new StringBuilder();
 
-            Word previous = null;
-            for (Word word : words) {
+            TextPositionSequence previous = null;
+            for (TextPositionSequence word : sequences) {
                 if (previous != null) {
                     if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');
@@ -217,8 +219,8 @@ public class TextPageBlock extends AbstractPageBlock {
     public int getNumberOfLines() {
 
         int numberOfLines = 1;
-        Word previous = null;
-        for (Word word : words) {
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sequences) {
             if (previous != null) {
                 if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
                     numberOfLines++;
@@ -260,17 +262,10 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-    public double getHighestFontSize() {
-
-        Double highest = frequencyCounters.getFontSizeFrequencyCounter().getHighest();
-        return highest == null ? 0 : highest;
-    }
-
-
     @Override
     public boolean isEmpty() {
 
-        return words.isEmpty();
+        return sequences.isEmpty();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j;
 @NoArgsConstructor
 @AllArgsConstructor
 @SuppressWarnings("pmd")
-public class Word extends TextBoundingBox implements CharSequence {
+public class TextPositionSequence extends TextBoundingBox implements CharSequence {
 
     public static final String STANDARD = "standard";
     public static final String BOLD_ITALIC = "bold, italic";
@@ -47,7 +47,7 @@ public class Word extends TextBoundingBox implements CharSequence {
     private Integer hashcodeCache;
 
 
-    public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
+    public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
 
         this.textPositions = textPositions.stream()
                 .map(RedTextPosition::fromTextPosition)
@@ -65,7 +65,7 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    public Word(List<RedTextPosition> textPositions, int page) {
+    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
 
         this.textPositions = textPositions;
         this.page = page;
@@ -98,9 +98,9 @@ public class Word extends TextBoundingBox implements CharSequence {
 
 
     @Override
-    public Word subSequence(int start, int end) {
+    public TextPositionSequence subSequence(int start, int end) {
 
-        var textPositionSequence = new Word();
+        var textPositionSequence = new TextPositionSequence();
         textPositionSequence.textPositions = textPositions.subList(start, end);
         textPositionSequence.page = page;
         textPositionSequence.dir = dir;
@@ -126,10 +126,10 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    public void add(Word word, RedTextPosition textPosition) {
+    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
 
         this.textPositions.add(textPosition);
-        this.page = word.getPage();
+        this.page = textPositionSequence.getPage();
         calculateBBoxAndHashcode();
     }
 
@@ -199,7 +199,7 @@ public class Word extends TextBoundingBox implements CharSequence {
         if (o == this) {
             return true;
         }
-        if (!(o instanceof Word other)) {
+        if (!(o instanceof TextPositionSequence other)) {
             return false;
         }
         if (!other.canEqual((Object) this)) {
@@ -220,7 +220,7 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    protected boolean canEqual(final Object other) {return other instanceof Word;}
+    protected boolean canEqual(final Object other) {return other instanceof TextPositionSequence;}
 
 
     public int hashCode() {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequenceComparator.java

@@ -3,19 +3,19 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 import java.util.Comparator;
 import java.util.HashMap;
 
-public class TextPositionSequenceComparator implements Comparator<Word> {
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
 
-    private HashMap<Word, TextBlockOnPage> lookup;
+    private HashMap<TextPositionSequence, TextBlockOnPage> lookup;
 
 
-    public TextPositionSequenceComparator(HashMap<Word, TextBlockOnPage> lookup) {
+    public TextPositionSequenceComparator(HashMap<TextPositionSequence, TextBlockOnPage> lookup) {
 
         this.lookup = lookup;
     }
 
 
     @Override
-    public int compare(Word number1, Word number2) {
+    public int compare(TextPositionSequence number1, TextPositionSequence number2) {
 
         int page1 = lookup.get(number1).page().getPageNumber();
         int page2 = lookup.get(number2).page().getPageNumber();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/BodyTextFrameService.java

@@ -5,9 +5,9 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/DividingColumnDetectionService.java

@@ -23,11 +23,11 @@ public class DividingColumnDetectionService {
     public List<Rectangle2D> detectColumns(PageContents pageContents) {
 
 
-        if (pageContents.getSortedWords().size() < 2) {
+        if (pageContents.getSortedTextPositionSequences().size() < 2) {
             return List.of(pageContents.getCropBox());
         }
 
-        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), pageContents.getCropBox());
+        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), pageContents.getCropBox());
 
         return detectColumnsFromLines(linesWithGapInformation.getXGaps(), pageContents.getCropBox());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/GapDetectionService.java

@@ -5,7 +5,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -18,23 +18,23 @@ public class GapDetectionService {
     private static final double NEW_LINE_FACTOR = 0.2;
 
 
-    public static GapInformation findGapsInLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public static GapInformation findGapsInLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        if (sortedWords.isEmpty()) {
+        if (sortedTextPositionSequences.isEmpty()) {
             return new GapInformation();
         }
 
-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
 
         XGapsContext xGapContext = XGapsContext.init(mainBodyTextFrame);
         YGapsContext yGapContext = YGapsContext.init(mainBodyTextFrame);
 
-        var previousTextPosition = sortedWords.get(0);
+        var previousTextPosition = sortedTextPositionSequences.get(0);
         Rectangle2D rectangle = toRectangle2D(previousTextPosition);
 
         xGapContext.addGapFromLeftEdgeOfMainBody(rectangle);
 
-        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
+        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
 
             double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
             double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
@@ -59,14 +59,14 @@ public class GapDetectionService {
             }
             previousTextPosition = currentTextPosition;
         }
-        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedWords.get(sortedWords.size() - 1)));
+        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedTextPositionSequences.get(sortedTextPositionSequences.size() - 1)));
         xGapContext.gapsPerLine.add(xGapContext.gapsInCurrentLine);
 
         return new GapInformation(xGapContext.gapsPerLine, yGapContext.gapsPerLine);
     }
 
 
-    private static Rectangle2D toRectangle2D(Word textPosition) {
+    private static Rectangle2D toRectangle2D(TextPositionSequence textPosition) {
 
         return mirrorY(textPosition.getBBox());
     }
@@ -87,18 +87,18 @@ public class GapDetectionService {
     }
 
 
-    private static void assertAllTextPositionsHaveSameDir(List<Word> words) {
+    private static void assertAllTextPositionsHaveSameDir(List<TextPositionSequence> textPositionSequences) {
 
-        assert words.stream()
-                .map(Word::getDir)
-                .allMatch(a -> a.equals(words.get(0).getDir()));
+        assert textPositionSequences.stream()
+                .map(TextPositionSequence::getDir)
+                .allMatch(a -> a.equals(textPositionSequences.get(0).getDir()));
     }
 
 
-    private static double getAvgTextPositionHeight(List<Word> words) {
+    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream()
-                .mapToDouble(Word::getHeight).average().orElseThrow();
+        return textPositionSequences.stream()
+                .mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/InvisibleTableDetectionService.java

@@ -7,17 +7,17 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public class InvisibleTableDetectionService {
 
-    public List<List<Rectangle2D>> detectTable(List<Word> words, Rectangle2D tableBBox) {
+    public List<List<Rectangle2D>> detectTable(List<TextPositionSequence> textPositionSequences, Rectangle2D tableBBox) {
 
-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(words);
-        GapInformation gaps = GapDetectionService.findGapsInLines(words, tableBBox);
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(textPositionSequences);
+        GapInformation gaps = GapDetectionService.findGapsInLines(textPositionSequences, tableBBox);
         List<Rectangle2D> gapsAcrossLines = GapsAcrossLinesService.detectXGapsAcrossLines(gaps, tableBBox);
         List<Double> columnXCoords = gapsAcrossLines.stream().map(RectangularShape::getCenterX).toList();
         int colCount = gapsAcrossLines.size();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/LineDetectionService.java

@@ -7,7 +7,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -19,37 +19,37 @@ public class LineDetectionService {
     private static final double X_GAP_FACTOR = 1; // multiplied with average text height, determines the minimum distance of gaps in lines
 
 
-    public LineInformation calculateLineInformation(List<Word> sortedWords) {
+    public LineInformation calculateLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
 
-        if (sortedWords.isEmpty()) {
+        if (sortedTextPositionSequences.isEmpty()) {
             return LineFactory.init().build();
         }
 
-        return buildLineInformation(sortedWords);
+        return buildLineInformation(sortedTextPositionSequences);
     }
 
 
-    public List<List<Rectangle2D>> findLinesWithGaps(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public List<List<Rectangle2D>> findLinesWithGaps(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        return calculateLineInformation(sortedWords).getBBoxWithGapsByLines();
+        return calculateLineInformation(sortedTextPositionSequences).getBBoxWithGapsByLines();
     }
 
 
-    public List<List<Word>> orderByLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public List<List<TextPositionSequence>> orderByLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        return calculateLineInformation(sortedWords).getSequencesByLines();
+        return calculateLineInformation(sortedTextPositionSequences).getSequencesByLines();
     }
 
 
-    private static LineInformation buildLineInformation(List<Word> sortedWords) {
+    private static LineInformation buildLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
 
-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
 
         LineFactory lineFactory = LineFactory.init();
 
-        var previousTextPosition = sortedWords.get(0);
+        var previousTextPosition = sortedTextPositionSequences.get(0);
         lineFactory.addToCurrentLine(previousTextPosition);
-        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
+        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
             if (isNewLine(currentTextPosition, previousTextPosition, avgTextPositionHeight) || isSplitByOrientation(currentTextPosition, previousTextPosition)) {
                 lineFactory.startNewLine();
             } else if (isXGap(currentTextPosition, previousTextPosition, avgTextPositionHeight)) {
@@ -63,25 +63,25 @@ public class LineDetectionService {
     }
 
 
-    private static double getAvgTextPositionHeight(List<Word> words) {
+    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream().mapToDouble(Word::getHeight).average().orElseThrow();
+        return textPositionSequences.stream().mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
     }
 
 
-    private static boolean isXGap(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isXGap(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
         return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
     }
 
 
-    private static boolean isSplitByOrientation(Word currentTextPosition, Word previousTextPosition) {
+    private static boolean isSplitByOrientation(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition) {
 
         return !previousTextPosition.getDir().equals(currentTextPosition.getDir());
     }
 
 
-    private static boolean isNewLine(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isNewLine(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
         return Math.abs(previousTextPosition.getYDirAdj() - currentTextPosition.getYDirAdj()) > avgTextPositionHeight;
     }
@@ -96,13 +96,13 @@ public class LineDetectionService {
         List<List<Rectangle2D>> bBoxWithGapsByLines;
         List<Rectangle2D> bBoxWithGapsInCurrentLine;
 
-        List<List<List<Word>>> sequencesWithGapsByLines;
-        List<List<Word>> sequencesWithGapsInCurrentLine;
+        List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+        List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine;
 
-        List<Word> currentSequencesWithoutGaps;
+        List<TextPositionSequence> currentSequencesWithoutGaps;
 
-        List<List<Word>> sequencesByLines;
-        List<Word> sequencesInCurrentLine;
+        List<List<TextPositionSequence>> sequencesByLines;
+        List<TextPositionSequence> sequencesInCurrentLine;
 
         List<List<Rectangle2D>> xGaps;
         List<List<Rectangle2D>> yGaps;
@@ -116,14 +116,14 @@ public class LineDetectionService {
             List<Rectangle2D> bBoxWithGapsInCurrentLine = new LinkedList<>();
             bBoxWithGapsByLines.add(bBoxWithGapsInCurrentLine);
 
-            List<List<List<Word>>> sequencesWithGapsByLines = new LinkedList<>();
-            List<List<Word>> sequencesWithGapsInCurrentLine = new LinkedList<>();
+            List<List<List<TextPositionSequence>>> sequencesWithGapsByLines = new LinkedList<>();
+            List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine = new LinkedList<>();
             sequencesWithGapsByLines.add(sequencesWithGapsInCurrentLine);
-            List<Word> currentSequencesWithoutGaps = new LinkedList<>();
+            List<TextPositionSequence> currentSequencesWithoutGaps = new LinkedList<>();
             sequencesWithGapsInCurrentLine.add(currentSequencesWithoutGaps);
 
-            List<List<Word>> sequencesByLines = new LinkedList<>();
-            List<Word> sequencesInCurrentLine = new LinkedList<>();
+            List<List<TextPositionSequence>> sequencesByLines = new LinkedList<>();
+            List<TextPositionSequence> sequencesInCurrentLine = new LinkedList<>();
             sequencesByLines.add(sequencesInCurrentLine);
 
             return new LineFactory(lineBBox,
@@ -178,13 +178,13 @@ public class LineDetectionService {
         }
 
 
-        private Rectangle2D textPositionBBox(List<Word> words) {
+        private Rectangle2D textPositionBBox(List<TextPositionSequence> textPositionSequences) {
 
-            return RectangleTransformations.rectangle2DBBox(words.stream().map(Word::getBBox).toList());
+            return RectangleTransformations.rectangle2DBBox(textPositionSequences.stream().map(TextPositionSequence::getBBox).toList());
         }
 
 
-        public void addToCurrentLine(Word current) {
+        public void addToCurrentLine(TextPositionSequence current) {
 
             sequencesInCurrentLine.add(current);
             currentSequencesWithoutGaps.add(current);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageContentExtractor.java

@@ -13,7 +13,7 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.core.io.ClassPathResource;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
@@ -40,7 +40,7 @@ public class PageContentExtractor {
                 stripper.setPdpage(pdPage);
                 stripper.getText(pdDocument);
 
-                Map<Float, List<Word>> sortedTextPositionSequencesPerDir = stripper.getWords()
+                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
                         .stream()
                         .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
 
@@ -57,7 +57,7 @@ public class PageContentExtractor {
     }
 
 
-    public List<Word> sortByDirAccordingToPageRotation(Map<Float, List<Word>> sortedTextPositionSequencesPerDir, int rotation) {
+    public List<TextPositionSequence> sortByDirAccordingToPageRotation(Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir, int rotation) {
 
         LinkedList<Float> sortedKeys = new LinkedList<>(sortedTextPositionSequencesPerDir.keySet().stream().sorted().toList());
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageInformationService.java

@@ -14,9 +14,9 @@ public class PageInformationService {
 
     public PageInformation build(PageContents pageContents) {
 
-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedWords());
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedTextPositionSequences());
         Rectangle2D mainBodyTextFrame = MainBodyTextFrameExtractionService.calculateMainBodyTextFrame(lineInformation);
-        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), mainBodyTextFrame);
+        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), mainBodyTextFrame);
 
         return new PageInformation(pageContents, lineInformation, mainBodyTextFrame, gapInformation);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -34,22 +34,7 @@ public class SimplifiedSectionTextService {
         List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
                 .flatMap(List::stream)
                 .collect(Collectors.toList());
-        return SimplifiedText.builder()
-                .numberOfPages(document.getNumberOfPages())
-                .sectionTexts(simplifiedText)
-                .mainSectionNumbers(document.getAllSections()
-                                            .stream()
-                                            .map(this::getSectionNumber)
-                                            .toList())
-                .headerSectionNumbers(document.getHeaders()
-                                              .stream()
-                                              .map(this::getSectionNumber)
-                                              .toList())
-                .footerSectionNumbers(document.getFooters()
-                                              .stream()
-                                              .map(this::getSectionNumber)
-                                              .toList())
-                .build();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
     }
 
 
@@ -64,13 +49,4 @@ public class SimplifiedSectionTextService {
                 .build();
     }
 
-
-    private String getSectionNumber(SemanticNode semanticNode) {
-
-        return semanticNode.getTreeId()
-                .stream()
-                .map(String::valueOf)
-                .collect(Collectors.joining("."));
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TextRulingsClassifier.java

@@ -5,7 +5,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -17,9 +17,9 @@ public class TextRulingsClassifier {
     private final static double TEXT_BBOX_THRESHOLD_FACTOR = 0.15; // multiplied with text width then subtracted from word width. If ruling covers this width, it is considered as strikethrough/underline.
 
 
-    public static void classifyUnderlinedAndStrikethroughText(List<Word> words, CleanRulings cleanRulings) {
+    public static void classifyUnderlinedAndStrikethroughText(List<TextPositionSequence> words, CleanRulings cleanRulings) {
 
-        for (Word word : words) {
+        for (TextPositionSequence word : words) {
             if (word.getDir().equals(TextDirection.ZERO) || word.getDir().equals(TextDirection.HALF_CIRCLE)) {
                 handleHorizontalText(cleanRulings, word);
             } else {
@@ -29,7 +29,7 @@ public class TextRulingsClassifier {
     }
 
 
-    private static void handleVerticalText(CleanRulings cleanRulings, Word word) {
+    private static void handleVerticalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
         float lowerY = (float) (word.getBBoxPdf().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
         float upperY = (float) (word.getBBoxPdf().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
@@ -63,7 +63,7 @@ public class TextRulingsClassifier {
     }
 
 
-    private static void handleHorizontalText(CleanRulings cleanRulings, Word word) {
+    private static void handleHorizontalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
         float leftX = (float) (word.getBBoxPdf().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
         float rightX = (float) (word.getBBoxPdf().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -7,17 +7,17 @@ import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Data;
 
@@ -222,14 +222,14 @@ public class BlockificationPostprocessingService {
             headline = sectionIdentifier + headline;
         }
 
-        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getWords(), headline);
+        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getSequences(), headline);
         if (wordSequenceResult.inSequence.isEmpty() && !headline.equals(title)) {
-            wordSequenceResult = findWordSequence(blockToSplit.getWords(), title);
+            wordSequenceResult = findWordSequence(blockToSplit.getSequences(), title);
         }
 
         boolean modifiedBlockToSplit = false;
         if (!wordSequenceResult.inSequence.isEmpty()) {
-            blockToSplit.setWords(wordSequenceResult.inSequence);
+            blockToSplit.setSequences(wordSequenceResult.inSequence);
             blockToSplit.recalculateBBox();
             modifiedBlockToSplit = true;
         }
@@ -250,19 +250,19 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static WordSequenceResult findWordSequence(List<Word> words, String text) {
+    private static WordSequenceResult findWordSequence(List<TextPositionSequence> textPositionSequences, String text) {
 
         String target = sanitizeString(text);
-        List<Word> inSequence = new ArrayList<>();
-        List<Word> preSequence = new ArrayList<>();
-        List<Word> postSequence = new ArrayList<>();
+        List<TextPositionSequence> inSequence = new ArrayList<>();
+        List<TextPositionSequence> preSequence = new ArrayList<>();
+        List<TextPositionSequence> postSequence = new ArrayList<>();
         StringBuilder currentSequence = new StringBuilder();
 
         if (target.isBlank()) {
             return new WordSequenceResult();
         }
 
-        for (Word sequence : words) {
+        for (TextPositionSequence sequence : textPositionSequences) {
 
             currentSequence.append(sanitizeString(sequence.toString()));
             inSequence.add(sequence);
@@ -274,10 +274,10 @@ public class BlockificationPostprocessingService {
                     int index = 0;
                     String toRemove = currentSequence.substring(0, currentSequence.length() - target.length());
 
-                    Word next = inSequence.get(index);
+                    TextPositionSequence next = inSequence.get(index);
                     while (currentSequence.length() - next.length() >= target.length()) {
 
-                        Word removed = inSequence.remove(index);
+                        TextPositionSequence removed = inSequence.remove(index);
                         currentSequence.delete(0, removed.toString().length());
                         preSequence.add(removed);
 
@@ -306,7 +306,7 @@ public class BlockificationPostprocessingService {
                 }
 
                 if (currentSequence.toString().equals(target)) {
-                    postSequence.addAll(words.subList(words.indexOf(sequence) + 1, words.size()));
+                    postSequence.addAll(textPositionSequences.subList(textPositionSequences.indexOf(sequence) + 1, textPositionSequences.size()));
                     return new WordSequenceResult(inSequence, preSequence, postSequence);
                 }
             }
@@ -316,10 +316,10 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static SplitSequenceResult splitSequence(Word sequence, String toRemove) {
+    private static SplitSequenceResult splitSequence(TextPositionSequence sequence, String toRemove) {
 
-        Word in = null;
-        Word out;
+        TextPositionSequence in = null;
+        TextPositionSequence out;
 
         String currentSequence = sequence.toString().toLowerCase(Locale.ROOT);
         int index = currentSequence.indexOf(toRemove);
@@ -337,9 +337,9 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static Word createSubSequence(Word sequence, int start, int end) {
+    private static TextPositionSequence createSubSequence(TextPositionSequence sequence, int start, int end) {
 
-        Word newSeq = new Word(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
+        TextPositionSequence newSeq = new TextPositionSequence(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
         newSeq.setParagraphStart(sequence.isParagraphStart());
         return newSeq;
     }
@@ -354,10 +354,10 @@ public class BlockificationPostprocessingService {
             List<TextPageBlock> mergedBlocks = new ArrayList<>();
             for (TextPageBlock textPageBlock : blocksToMerge.subList(1, blocksToMerge.size())) {
 
-                if (firstBlock != null && !firstBlock.getWords().isEmpty()) {
+                if (firstBlock != null && !firstBlock.getSequences().isEmpty()) {
 
                     if (textPageBlock.getDir() == firstBlock.getDir()) {
-                        firstBlock.addAll(textPageBlock.getWords());
+                        firstBlock.addAll(textPageBlock.getSequences());
                         mergedBlocks.add(textPageBlock);
                     }
                 }
@@ -467,9 +467,7 @@ public class BlockificationPostprocessingService {
 
     private static String sanitizeString(String text) {
 
-        return TextNormalizationUtilities.removeAllWhitespaces(text)//
-                .trim() // sometimes there are trailing empty bytes at the end of the string trim() seems to remove them
-                .toLowerCase(Locale.ENGLISH);
+        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
     }
 
 
@@ -496,12 +494,12 @@ public class BlockificationPostprocessingService {
 
     public static class WordSequenceResult {
 
-        public List<Word> inSequence;
-        public List<Word> preSequence;
-        public List<Word> postSequence;
+        public List<TextPositionSequence> inSequence;
+        public List<TextPositionSequence> preSequence;
+        public List<TextPositionSequence> postSequence;
 
 
-        public WordSequenceResult(List<Word> inSequence, List<Word> preSequence, List<Word> postSequence) {
+        public WordSequenceResult(List<TextPositionSequence> inSequence, List<TextPositionSequence> preSequence, List<TextPositionSequence> postSequence) {
 
             this.inSequence = inSequence;
             this.preSequence = preSequence;
@@ -522,7 +520,7 @@ public class BlockificationPostprocessingService {
 
     }
 
-    public record SplitSequenceResult(Word in, Word out) {
+    public record SplitSequenceResult(TextPositionSequence in, TextPositionSequence out) {
 
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -6,7 +6,7 @@ import java.util.ListIterator;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 import lombok.RequiredArgsConstructor;
@@ -30,7 +30,7 @@ public class DocstrumBlockificationService {
     static final float THRESHOLD = 1f;
 
 
-    public ClassificationPage blockify(List<Word> textPositions,
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions,
                                        CleanRulings rulings,
                                        boolean xyOrder,
                                        LayoutDebugLayer visualizations,
@@ -72,16 +72,16 @@ public class DocstrumBlockificationService {
         List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         zones.forEach(zone -> {
 
-            List<Word> words = new ArrayList<>();
+            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
             zone.getLines()
                     .forEach(line -> {
                         line.getWords()
                                 .forEach(word -> {
-                                    words.add(new Word(word.getTextPositions(), word.getPage()));
+                                    textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), word.getPage()));
                                 });
                     });
 
-            abstractPageBlocks.add(buildTextBlock(words, 0));
+            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
         });
 
         return abstractPageBlocks;
@@ -102,7 +102,7 @@ public class DocstrumBlockificationService {
             }
             TextPageBlock current = (TextPageBlock) block;
 
-            if (previous != null && !previous.getWords().isEmpty()) {
+            if (previous != null && !previous.getSequences().isEmpty()) {
 
                 if (current.getDir() != previous.getDir() || usedRulings.lineBetween(current, previous)) {
                     previous = current;
@@ -182,8 +182,8 @@ public class DocstrumBlockificationService {
 
     private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
 
-        previous.addAll(current.getWords());
-        previous = buildTextBlock(previous.getWords(), 0);
+        previous.addAll(current.getSequences());
+        previous = buildTextBlock(previous.getSequences(), 0);
         previous.setToDuplicate(toDuplicate);
         if (current.getClassification() != null && previous.getClassification() == null) {
             previous.setClassification(current.getClassification());
@@ -283,8 +283,8 @@ public class DocstrumBlockificationService {
                 if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold)) {
 
                     boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getWords());
-                    current = buildTextBlock(current.getWords(), 0);
+                    current.addAll(inner.getSequences());
+                    current = buildTextBlock(current.getSequences(), 0);
 
                     current.setToDuplicate(toDuplicate);
                     blocks.set(i, null);
@@ -301,7 +301,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         return new TextPageBlock(wordBlockList);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -16,14 +16,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 @SuppressWarnings("all")
 @Service
 public class DocuMineBlockificationService {
 
     static final float THRESHOLD = 1f;
-    public static final double FONT_SIZE_CHANGE_RATIO = 0.15;
 
     Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
 
@@ -37,9 +36,9 @@ public class DocuMineBlockificationService {
      * @param cleanRulings  All rulings on a page
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings) {
 
-        List<Word> chunkWords = new ArrayList<>();
+        List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> textPageBlocks = new ArrayList<>();
 
         CleanRulings usedRulings = cleanRulings.withoutTextRulings();
@@ -48,11 +47,11 @@ public class DocuMineBlockificationService {
         double maxX = 0;
         double minY = 1000;
         double maxY = 0;
-        Word prev = null;
+        TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
         Double splitX1 = null;
-        for (Word word : textPositions) {
+        for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = prev != null && word.getYDirAdj() - prev.getMaxYDirAdj() > Math.min(word.getHeight(), prev.getHeight()) * 1.1;
             boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
@@ -61,7 +60,11 @@ public class DocuMineBlockificationService {
             boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
             boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && isFontChange(word, prev);
+            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 //
+                                                && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
+                                                    || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
+                                                    || Math.abs(prev.getFontSize() - word.getFontSize()) >= 1
+                                                    || Math.abs(word.getTextHeight() - prev.getTextHeight()) > 0.8);
 
             Matcher matcher = pattern.matcher(chunkWords.stream()
                                                       .collect(Collectors.joining(" ")).toString());
@@ -124,15 +127,6 @@ public class DocuMineBlockificationService {
     }
 
 
-    private static boolean isFontChange(Word word, Word prev) {
-
-        return word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
-               || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
-               || Math.abs(prev.getFontSize() - word.getFontSize()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getFontSize(), word.getFontSize())
-               || Math.abs(word.getTextHeight() - prev.getTextHeight()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getTextHeight(), word.getTextHeight());
-    }
-
-
     public void mergeblocks(ClassificationPage page, CleanRulings usedRulings, float xThreshold, float yThreshold) {
 
         var blocks = page.getTextBlocks();
@@ -175,8 +169,8 @@ public class DocuMineBlockificationService {
                         .equals(inner.getClassification()))) {
 
                     boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getWords());
-                    current = buildTextBlock(current.getWords(), 0);
+                    current.addAll(inner.getSequences());
+                    current = buildTextBlock(current.getSequences(), 0);
                     current.setClassification(inner.getClassification());
                     current.setToDuplicate(toDuplicate);
                     blocks.set(i, null);
@@ -199,7 +193,7 @@ public class DocuMineBlockificationService {
     }
 
 
-    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         return new TextPageBlock(wordBlockList);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -11,7 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 @SuppressWarnings("all")
@@ -30,20 +30,20 @@ public class RedactManagerBlockificationService {
      * @param visualizations
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
 
         CleanRulings usedRulings = cleanRulings.withoutTextRulings();
 
         int indexOnPage = 0;
-        List<Word> chunkWords = new ArrayList<>();
+        List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList = new ArrayList<>();
 
         double minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        Word prev = null;
+        TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
         Double splitX1 = null;
-        for (Word word : textPositions) {
+        for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = word.getYDirAdj() - maxY > word.getHeight() * 1.25;
             boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -10,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -72,16 +73,14 @@ public class ClarifyndClassificationService {
             && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-                headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                document.setHeadlines(true);
+                textBlock.setClassification(PageBlockType.TITLE);
             }
         } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                    && PositionUtils.getApproxLineCount(textBlock) < 4.9
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -91,7 +90,7 @@ public class ClarifyndClassificationService {
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationPatterns.java

@@ -1,29 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.regex.Pattern;
-
-public class ClassificationPatterns {
-
-    public static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s(?:14C)?\\s*[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
-
-    public static final Pattern AMOUNT_PATTERN = Pattern.compile(
-            "^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|ul|μl|l|ug|μg|g|kg|mg|cm|cm2|cm3|mm|mm2|mm3|km|km2|m|m2|m3|lb|oz|ppm|dpm|days|weeks|months|%|f|ppb)\\b",
-            Pattern.CASE_INSENSITIVE);
-
-
-
-    public static final Pattern TABLE_OR_FIGURE_PATTERN = Pattern.compile(
-            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
-            Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
-
-    public static final Pattern NUMERIC = Pattern.compile("[0-9]+");
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationService.java

@@ -4,9 +4,11 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -1,20 +1,11 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.classification;
 
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.ALPHANUMERIC;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AT_LEAST_3_CHARS_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_PATTERN;
-
 import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
@@ -22,60 +13,90 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.HeaderFooterDetection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocuMineClassificationService {
 
+    private static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+    public static final Pattern AMOUNT_PATTERN = Pattern.compile("^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|l|g|kg|mg|cm|cm2|cm3|mm|mm2|mm3|km|km2|m|m2|m3|lb|oz|ppm|dpm|days|weeks|months|%|f)\\b",
+                                                                  Pattern.CASE_INSENSITIVE);
+    private static final Pattern TABLE_OR_FIGURE_PATTERN = Pattern.compile(
+            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+    private static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
+
     public static final int SEPARATION_THRESHOLD = 10; // if the min distance between a textblock and all its surrounding blocks, the regexes can be more lenient.
     public static final int SURROUNDING_BLOCKS_RADIUS = 3; // number of surrounding blocks before and after the current textblock to be tested
 
-    ListItemClassificationService listItemClassificationService;
-
 
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Double> headlineFontSizes = buildHeadlineFontSizes(document);
-        List<AbstractBlockOnPage> blocks = buildBlocksPerPage(document);
-        log.debug("Headline FontSizes are: {}", headlineFontSizes);
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
+
+        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
         HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();
 
-        for (int i = 0; i < blocks.size(); i++) {
-            AbstractBlockOnPage block = blocks.get(i);
-            document.getLayoutDebugLayer().addTextBlockVisualizations(block.page().getTextBlocks(), block.page().getPageNumber());
-            classifyBlock(headlineClassificationService, i, blocks, document, headlineFontSizes);
+        for (ClassificationPage page : document.getPages()) {
+            document.getLayoutDebugLayer().addTextBlockVisualizations(page.getTextBlocks(), page.getPageNumber());
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
         }
+    }
 
+
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {
+
+        List<AbstractPageBlock> textBlocks = page.getTextBlocks();
+        for (int i = 0; i < textBlocks.size(); i++) {
+            AbstractPageBlock textBlock = textBlocks.get(i);
+            if (textBlock instanceof TextPageBlock) {
+                List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocks(i, textBlocks);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, surroundingBlocks, page, document, headlineFontSizes);
+            }
+        }
+    }
+
+
+    private List<AbstractPageBlock> getSurroundingBlocks(int originalIndex, List<AbstractPageBlock> textBlocks) {
+
+        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
+        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
+        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
+        for (int i = start; i < end; i++) {
+            if (i == originalIndex) {
+                continue;
+            }
+            if (textBlocks.get(i).getText().length() <= 1) {
+                continue;
+            }
+            surroundingBlocks.add(textBlocks.get(i));
+        }
+        return surroundingBlocks;
     }
 
 
     private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               int currentIndex,
-                               List<AbstractBlockOnPage> allBlocks,
+                               TextPageBlock textBlock,
+                               List<AbstractPageBlock> surroundingBlocks,
+                               ClassificationPage page,
                                ClassificationDocument document,
                                List<Double> headlineFontSizes) {
 
-        TextPageBlock textBlock;
-        if (allBlocks.get(currentIndex).block() instanceof TextPageBlock block) {
-            textBlock = block;
-        } else {
-            return;
-        }
-        ClassificationPage page = allBlocks.get(currentIndex).page();
-        List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocksOnPage(currentIndex, allBlocks);
-
         log.debug("headlineFontSizes: {}", headlineFontSizes);
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -92,9 +113,6 @@ public class DocuMineClassificationService {
 
         boolean enoughChars = charCount > textBlock.getText().length() * 0.5;
 
-        List<ListIdentifier> listIdentifiers = listItemClassificationService.findConfirmedListIdentifiers(currentIndex, allBlocks);
-        document.getLayoutDebugLayer().addListIdentifiers(listIdentifiers);
-
         if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
             headlineClassificationService.setLastHeadlineFromOutline(textBlock);
             return;
@@ -108,21 +126,26 @@ public class DocuMineClassificationService {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
-        if (textBlock.getText().length() > 5
-            && greaterOrEqualFontThanDocumentAverage(textBlock, document)
-            && PositionUtils.getApproxLineCount(textBlock) < 5.9
-            && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
-                && Character.isDigit(textBlock.toString().charAt(0)) //
-                && isAtLeast3Characters //
-                && !textBlock.toString().contains(":") //
-                || textBlock.toString().startsWith("APPENDIX") //
-                || textBlock.toString().startsWith("FIGURE") //
-                || textBlock.toString().startsWith("Continued TABLE") //
-                || textBlock.toString().startsWith("TABLE"))
-            && !textBlock.toString().endsWith(":")
-            && isAtLeast3Characters
-            && !isAmount
-            && enoughChars) {
+        if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                                          && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+                textBlock.setClassification(PageBlockType.TITLE);
+            }
+        } else if (textBlock.getText().length() > 5
+                   && greaterOrEqualFontThanDocumentAverage(textBlock, document)
+                   && PositionUtils.getApproxLineCount(textBlock) < 5.9
+                   && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
+                       && Character.isDigit(textBlock.toString().charAt(0)) //
+                       && isAtLeast3Characters //
+                       && !textBlock.toString().contains(":") //
+                       || textBlock.toString().startsWith("APPENDIX") //
+                       || textBlock.toString().startsWith("FIGURE") //
+                       || textBlock.toString().startsWith("Continued TABLE") //
+                       || textBlock.toString().startsWith("TABLE"))
+                   && !textBlock.toString().endsWith(":")
+                   && isAtLeast3Characters
+                   && !isAmount
+                   && enoughChars) {
 
             setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
         } else if (isAllCaps(textBlock)
@@ -147,24 +170,11 @@ public class DocuMineClassificationService {
         } else if (hasSeparation(textBlock, surroundingBlocks)//
                    && greaterOrEqualFontThanPageAverage(textBlock, page)//
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9//
-                   && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
+                   && (tableOrFigureMatcher.reset().find() || headlineWithSingleIdentifierMatcher.reset().find())//
                    && !isAmount//
                    && !headlineWithSlashesMatches) {
 
             setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-//        } else if (textBlock.getMostPopularWordFont().contains("bold")
-//                   && greaterOrEqualFontThanPageAverage(textBlock, page)
-//                   && textBlock.getWords().size() <= 6
-//                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-//                   && isAtLeast3Characters
-//                   && charCount > textBlock.getText().length() * 0.75
-//                   && !textBlock.getText().contains(":")
-//                   && textBlock.getWidth() < page.getBodyTextFrame().getWidth() * 0.7) {
-//
-//            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (!listIdentifiers.isEmpty()) {
-
-            textBlock.setClassification(PageBlockType.LIST_ITEM);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("bold")
@@ -254,92 +264,6 @@ public class DocuMineClassificationService {
         document.setHeadlines(true);
     }
 
-
-    private List<AbstractBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
-
-        List<AbstractBlockOnPage> blocks = new ArrayList<>();
-        for (ClassificationPage page : document.getPages()) {
-            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
-                if (abstractPageBlock instanceof TextPageBlock textBlock) {
-                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
-                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
-                        continue;
-                    }
-                    blocks.add(new AbstractBlockOnPage(textBlock, page));
-                }
-            }
-        }
-        return blocks;
-    }
-
-
-    private static List<Double> buildHeadlineFontSizes(ClassificationDocument document) {
-
-        if (document.getFontSizeCounter().getCountPerValue().size() <= 6) {
-            return document.getFontSizeCounter().getValuesInReverseOrder();
-        }
-
-        List<Map.Entry<Double, Integer>> sortedEntries = new ArrayList<>(document.getFontSizeCounter().getCountPerValue().entrySet());
-        sortedEntries.sort(Map.Entry.comparingByKey());
-
-        int totalCount = sortedEntries.stream()
-                .mapToInt(Map.Entry::getValue).sum();
-
-        int cumulativeCount = 0;
-        Iterator<Map.Entry<Double, Integer>> iterator = sortedEntries.iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<Double, Integer> entry = iterator.next();
-            cumulativeCount += entry.getValue();
-            if (cumulativeCount > totalCount * 0.3) {
-                break; // We've filtered the bottom 30%, so stop.
-            }
-            iterator.remove();
-        }
-
-        if (sortedEntries.size() < 6) {
-            return document.getFontSizeCounter().getValuesInReverseOrder();
-        }
-        int clusterSize = Math.max(1, sortedEntries.size() / 6);
-
-        List<List<Double>> clusters = new ArrayList<>();
-        for (int i = 0; i < 6; i++) {
-            clusters.add(new ArrayList<>());
-        }
-
-        for (int i = 0; i < sortedEntries.size(); i++) {
-            int clusterIndex = Math.min(i / clusterSize, 5);
-            clusters.get(clusterIndex).add(sortedEntries.get(i).getKey());
-        }
-
-        return clusters.stream()
-                .map(cluster -> cluster.stream()
-                        .mapToDouble(d -> d).average()
-                        .orElseThrow())
-                .sorted(Comparator.reverseOrder())
-                .toList();
-    }
-
-
-    private List<AbstractPageBlock> getSurroundingBlocksOnPage(int originalIndex, List<AbstractBlockOnPage> textBlocks) {
-
-        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
-        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
-        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
-        for (int i = start; i < end; i++) {
-            if (i == originalIndex) {
-                continue;
-            }
-            if (textBlocks.get(i).block().getText().length() <= 1) {
-                continue;
-            }
-            if (!textBlocks.get(i).page().equals(textBlocks.get(originalIndex).page())) {
-                continue;
-            }
-            surroundingBlocks.add(textBlocks.get(i).block());
-        }
-        return surroundingBlocks;
-    }
-
 }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/HeadlineClassificationService.java

@@ -73,18 +73,13 @@ public class HeadlineClassificationService {
 
     public static PageBlockType headlineClassByFontSize(TextPageBlock textBlock, List<Double> fontSizeGroups) {
 
-        List<Double> distances = fontSizeGroups.stream()
-                .map(fontSize -> Math.abs(fontSize - textBlock.getMostPopularWordFontSize()))
-                .toList();
-        double min = Double.MAX_VALUE;
-        int argMin = -1;
-        for (int i = 0; i < distances.size(); i++) {
-            if (distances.get(i) < min) {
-                min = distances.get(i);
-                argMin = i;
+        PageBlockType headlineType = PageBlockType.H1;
+        for (int i = 1; i <= fontSizeGroups.size(); i++) {
+            if (textBlock.getMostPopularWordFontSize() == fontSizeGroups.get(i - 1)) {
+                headlineType = PageBlockType.getHeadlineType(i);
             }
         }
-        return PageBlockType.getHeadlineType(argMin);
+        return headlineType;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ListItemClassificationService.java

@@ -1,99 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-
-@Service
-public class ListItemClassificationService {
-
-    public static final int LIST_IDENTIFIER_SEARCH_RADIUS = 3;
-
-
-    public List<ListIdentifier> findConfirmedListIdentifiers(int currentIndex, List<AbstractBlockOnPage> allBlocks) {
-
-        List<ListIdentifier> listIdentifiers = extractListIdentifiers(allBlocks.get(currentIndex));
-        if (listIdentifiers.isEmpty()) {
-            return Collections.emptyList();
-        }
-        if (listIdentifiers.size() > 1 && ListIdentifier.isInOrder(listIdentifiers)) {
-            return listIdentifiers;
-        }
-
-        int start = Math.max(0, currentIndex - LIST_IDENTIFIER_SEARCH_RADIUS);
-        int end = Math.min(allBlocks.size(), currentIndex + LIST_IDENTIFIER_SEARCH_RADIUS);
-
-        List<ListIdentifier> identifiersBehind = new ArrayList<>();
-        if (start < currentIndex) {
-            identifiersBehind.addAll(allBlocks.subList(start, currentIndex)
-                                             .stream()
-                                             .map(this::extractListIdentifiers)
-                                             .flatMap(Collection::stream)
-                                             .toList());
-        }
-        if (!identifiersBehind.isEmpty()) {
-            listIdentifiers.add(0, identifiersBehind.get(identifiersBehind.size() - 1));
-            if (ListIdentifier.isInOrder(listIdentifiers)) {
-                return listIdentifiers;
-            }
-            listIdentifiers.remove(0);
-        }
-        List<ListIdentifier> identifiersAhead = new ArrayList<>();
-        if (currentIndex + 1 < end) {
-            identifiersAhead.addAll(allBlocks.subList(currentIndex + 1, end)
-                                            .stream()
-                                            .map(this::extractListIdentifiers)
-                                            .flatMap(Collection::stream)
-                                            .toList());
-        }
-        if (!identifiersAhead.isEmpty()) {
-            listIdentifiers.add(identifiersAhead.get(0));
-            if (ListIdentifier.isInOrder(listIdentifiers)) {
-                return listIdentifiers;
-            }
-            listIdentifiers.remove(listIdentifiers.size() - 1);
-        }
-        return Collections.emptyList();
-
-    }
-
-
-    private List<ListIdentifier> extractListIdentifiers(AbstractBlockOnPage block) {
-
-        List<ListIdentifier> result = new LinkedList<>();
-        if (block.block() instanceof TextPageBlock textBlock) {
-            List<Word> sequences = textBlock.getWords();
-            for (int i = 0; i < sequences.size(); i++) {
-
-                if (i != 0 && sequences.get(i - 1).getXDirAdj() < sequences.get(i).getXDirAdj()) {
-                    // is not the start of a line, continue
-                    continue;
-                }
-
-                Word sequence = sequences.get(i);
-                List<Word> wordsAtStartOfLine = new ArrayList<>(3);
-                int end = Math.min(sequences.size(), i + 3);
-                for (int j = i; j < end; j++) {
-                    if (sequences.get(j).intersectsYDirAdj(sequence, 2)) {
-                        wordsAtStartOfLine.add(sequences.get(j));
-                    } else {
-                        break;
-                    }
-                }
-
-                ListIdentifier.parse(wordsAtStartOfLine, block.page().getPageNumber()).ifPresent(result::add);
-            }
-        }
-        return result;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -11,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -85,16 +86,14 @@ public class RedactManagerClassificationService {
                                           && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
 
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-                headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                document.setHeadlines(true);
+                textBlock.setClassification(PageBlockType.TITLE);
             }
         } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                    && PositionUtils.getApproxLineCount(textBlock) < 4.9
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -104,7 +103,7 @@ public class RedactManagerClassificationService {
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TableOfContentsClassificationService.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.classification;
 
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.NUMERIC;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService.AMOUNT_PATTERN;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -14,6 +13,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
@@ -25,7 +25,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequenceComparator;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
@@ -42,6 +42,8 @@ public class TableOfContentsClassificationService {
     public static final int INTERSECTION_TOLERANCE = 2; // equality threshold for x intersection
     public static final int DENSITY_THRESHOLD_COUNT = 10; // describes the minimum density, at least this many entries per page height are required
 
+    private static final Pattern NUMERIC = Pattern.compile("[0-9]+");
+
 
     @SuppressWarnings("checkstyle:ModifiedControlVariable")
     public void classifyTableOfContents(ClassificationDocument document) {
@@ -55,13 +57,11 @@ public class TableOfContentsClassificationService {
                 continue;
             }
 
-            int end = identifyTOCItems(i + 1, textBlocks, document);
+            int offset = identifyTOCItems(i + 1, textBlocks, document);
 
-            if (end > i + 1) {
-                if (textBlock.textBlock().getClassification() == null) {
-                    textBlock.textBlock().setClassification(PageBlockType.H1);
-                }
-                i = end;
+            if (offset > 1) {
+                textBlock.textBlock().setClassification(PageBlockType.H1);
+                i += offset;
             }
         }
     }
@@ -71,8 +71,8 @@ public class TableOfContentsClassificationService {
 
         ClassificationPage startPage = textBlocks.get(start).page();
         List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
-        HashMap<Word, TextBlockOnPage> lookup = new HashMap<>();
-        List<Word> numbers = extractNumbers(initialLookAhead, lookup, document.getPages().size());
+        HashMap<TextPositionSequence, TextBlockOnPage> lookup = new HashMap<>();
+        List<TextPositionSequence> numbers = extractNumbers(initialLookAhead, lookup, document.getPages().size());
         TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, lookup);
 
         int lastCandidate = start;
@@ -93,9 +93,9 @@ public class TableOfContentsClassificationService {
                 break;
             }
 
-            List<Word> numbersFromBlock = extractNumbers(textBlockOnPage, lookup, document.getPages().size());
+            List<TextPositionSequence> numbersFromBlock = extractNumbers(textBlockOnPage, lookup, document.getPages().size());
 
-            List<Word> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
+            List<TextPositionSequence> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
 
             if (currentRightmostCluster.size() < MINIMUM_MATCHES) {
                 log.debug("No numbers indicating a table of contents here.");
@@ -132,7 +132,7 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static void addVisualization(LayoutDebugLayer layoutDebugLayer, TocNumberFinder tocNumberFinder, Map<Word, TextBlockOnPage> lookup) {
+    private static void addVisualization(LayoutDebugLayer layoutDebugLayer, TocNumberFinder tocNumberFinder, Map<TextPositionSequence, TextBlockOnPage> lookup) {
 
         tocNumberFinder.getCurrentRightmostCluster()
                 .stream()
@@ -141,9 +141,9 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static boolean anyIntersection(Collection<Word> numbers1,
-                                           Collection<Word> numbers2,
-                                           Map<Word, TextBlockOnPage> lookup) {
+    private static boolean anyIntersection(Collection<TextPositionSequence> numbers1,
+                                           Collection<TextPositionSequence> numbers2,
+                                           Map<TextPositionSequence, TextBlockOnPage> lookup) {
 
         return numbers1.stream()
                 .anyMatch(numberFromCluster -> numbers2.stream()
@@ -151,9 +151,9 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static List<Word> extractNumbers(List<TextBlockOnPage> textBlocks, Map<Word, TextBlockOnPage> lookup, int numberOfPages) {
+    private static List<TextPositionSequence> extractNumbers(List<TextBlockOnPage> textBlocks, Map<TextPositionSequence, TextBlockOnPage> lookup, int numberOfPages) {
 
-        List<Word> blocks = new LinkedList<>();
+        List<TextPositionSequence> blocks = new LinkedList<>();
         for (TextBlockOnPage textBlock : textBlocks) {
             blocks.addAll(extractNumbers(textBlock, lookup, numberOfPages));
         }
@@ -161,14 +161,14 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static List<Word> extractNumbers(TextBlockOnPage textBlock, Map<Word, TextBlockOnPage> lookup, int numberOfPages) {
+    private static List<TextPositionSequence> extractNumbers(TextBlockOnPage textBlock, Map<TextPositionSequence, TextBlockOnPage> lookup, int numberOfPages) {
 
-        List<Word> blocks = new LinkedList<>();
+        List<TextPositionSequence> blocks = new LinkedList<>();
         TextPageBlock block = textBlock.textBlock();
-        List<Word> sequences = block.getWords();
+        List<TextPositionSequence> sequences = block.getSequences();
         for (int i = 0; i < sequences.size(); i++) {
 
-            Word word = sequences.get(i);
+            TextPositionSequence word = sequences.get(i);
 
             if (!NUMERIC.matcher(word).matches()) {
                 continue;
@@ -193,17 +193,17 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static CharSequence getSurroundingString(int i, List<Word> sequences) {
+    private static CharSequence getSurroundingString(int i, List<TextPositionSequence> sequences) {
 
         int end = Math.min(i + 5, sequences.size());
         return sequences.subList(i, end)
                 .stream()
-                .map(Word::toString)
+                .map(TextPositionSequence::toString)
                 .collect(Collectors.joining(" "));
     }
 
 
-    private static boolean matches(Word number1, Word number2, Map<Word, TextBlockOnPage> lookup) {
+    private static boolean matches(TextPositionSequence number1, TextPositionSequence number2, Map<TextPositionSequence, TextBlockOnPage> lookup) {
 
         if (number1.getDir() != number2.getDir()) {
             return false;
@@ -237,7 +237,7 @@ public class TableOfContentsClassificationService {
                                                                   || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
                         continue;
                     }
-                    blocks.add(new TextBlockOnPage(textBlock, page));
+                    blocks.add(new TextBlockOnPage(page, textBlock));
                 }
             }
         }
@@ -247,11 +247,11 @@ public class TableOfContentsClassificationService {
 
     private static class TocNumberFinder {
 
-        final UnionFind<Word> numberClusters;
-        final HashMap<Word, TextBlockOnPage> lookup;
+        final UnionFind<TextPositionSequence> numberClusters;
+        final HashMap<TextPositionSequence, TextBlockOnPage> lookup;
 
 
-        TocNumberFinder(List<Word> blocks, HashMap<Word, TextBlockOnPage> lookup) {
+        TocNumberFinder(List<TextPositionSequence> blocks, HashMap<TextPositionSequence, TextBlockOnPage> lookup) {
 
             this.numberClusters = new UnionFind<>(new HashSet<>(blocks));
             for (int i = 0; i < blocks.size(); i++) {
@@ -265,14 +265,14 @@ public class TableOfContentsClassificationService {
         }
 
 
-        public void add(Word number) {
+        public void add(TextPositionSequence number) {
 
             if (numberClusters.getElements().contains(number)) {
                 return;
             }
 
             numberClusters.addElement(number);
-            for (Word element : numberClusters.getElements()) {
+            for (TextPositionSequence element : numberClusters.getElements()) {
                 if (matches(number, element, lookup)) {
                     numberClusters.union(element, number);
                 }
@@ -280,7 +280,7 @@ public class TableOfContentsClassificationService {
         }
 
 
-        public List<Word> getCurrentRightmostCluster() {
+        public List<TextPositionSequence> getCurrentRightmostCluster() {
 
             return numberClusters.getGroups()
                     .stream()
@@ -322,9 +322,9 @@ public class TableOfContentsClassificationService {
 //        }
 
 
-        public List<Word> removeOutliers(List<Word> numbers) {
+        public List<TextPositionSequence> removeOutliers(List<TextPositionSequence> numbers) {
 
-            List<Word> result = new ArrayList<>();
+            List<TextPositionSequence> result = new ArrayList<>();
 
             result.add(numbers.get(0));
 
@@ -346,13 +346,13 @@ public class TableOfContentsClassificationService {
 
 
         // Helper method to check if removing the current number results in a better order
-        public static boolean isBetterWithout(List<Word> numbers, int i) {
+        public static boolean isBetterWithout(List<TextPositionSequence> numbers, int i) {
 
             if (i == 0 || i == numbers.size() - 1) {
                 return false;
             }
 
-            int prev = getNumberAsInt(numbers, i - 1);
+            int prev = getNumberAsInt(numbers, i);
             int curr = getNumberAsInt(numbers, i);
             int next = getNumberAsInt(numbers, i + 1);
 
@@ -362,7 +362,7 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static int getNumberAsInt(List<Word> numbers, int i) {
+    private static int getNumberAsInt(List<TextPositionSequence> numbers, int i) {
 
         return Integer.parseInt(numbers.get(i).toString());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -16,8 +16,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
@@ -65,7 +65,7 @@ public class DocumentGraphFactory {
 
         document.getPages()
                 .forEach(context::buildAndAddPageWithCounter);
-        addSectionsForToC(layoutParsingType, document, context, documentGraph);
+        addSections(layoutParsingType, document, context, documentGraph);
         addHeaderAndFooterToEachPage(document, context);
 
         documentGraph.setNumberOfPages(context.pages.size());
@@ -92,17 +92,17 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addSectionsForToC(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
+    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
 
         for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
             GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
             Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
-                                                                                  parent,
-                                                                                  tocItem.getChildren().isEmpty(),
-                                                                                  tocItem.getNonEmptySectionBlocks(),
-                                                                                  tocItem.getImages(),
-                                                                                  context,
-                                                                                  document);
+                                                                                   parent,
+                                                                                   tocItem.getChildren().isEmpty(),
+                                                                                   tocItem.getNonEmptySectionBlocks(),
+                                                                                   tocItem.getImages(),
+                                                                                   context,
+                                                                                   document);
             tocItem.setSection(section.orElse(null));
         }
     }
@@ -133,9 +133,9 @@ public class DocumentGraphFactory {
 
         if (node instanceof DuplicatedParagraph duplicatedParagraph) {
             AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                      .flatMap(tb -> tb.getWords()
-                                                                                                              .stream())
-                                                                                                      .collect(Collectors.toList()), node, context, page);
+                                                                                                       .flatMap(tb -> tb.getSequences()
+                                                                                                               .stream())
+                                                                                                       .collect(Collectors.toList()), node, context, page);
             duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
         }
 
@@ -260,6 +260,7 @@ public class DocumentGraphFactory {
         DocumentTree documentTree;
         Map<Page, Integer> pages;
         List<AbstractSemanticNode> sections;
+        List<ClassifiedImage> images;
         TextBlockFactory textBlockFactory;
 
 
@@ -268,6 +269,7 @@ public class DocumentGraphFactory {
             documentTree = new DocumentTree(document);
             pages = new HashMap<>();
             sections = new LinkedList<>();
+            images = new LinkedList<>();
             textBlockFactory = new TextBlockFactory();
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -12,7 +12,7 @@ import java.util.Objects;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -28,7 +28,7 @@ public class SearchTextWithTextPositionFactory {
     public static final double LINEBREAK_DELTA_TOLERANCE = 1.5;
 
 
-    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<Word> sequences) {
+    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<TextPositionSequence> sequences) {
 
         if (sequences.isEmpty() || sequences.stream()
                 .allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@@ -40,7 +40,7 @@ public class SearchTextWithTextPositionFactory {
         RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
         RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();
 
-        for (Word word : sequences) {
+        for (TextPositionSequence word : sequences) {
             for (int i = 0; i < word.getTextPositions().size(); ++i) {
 
                 currentTextPosition = word.getTextPositions().get(i);
@@ -66,7 +66,7 @@ public class SearchTextWithTextPositionFactory {
         }
 
         List<Rectangle2D> positions = sequences.stream()
-                .map(Word::getTextPositions)
+                .map(TextPositionSequence::getTextPositions)
                 .flatMap(Collection::stream)
                 .map(RedTextPosition::getBBoxPdf)
                 .toList();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -10,7 +10,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TableNodeFactory.java

@@ -6,7 +6,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
@@ -16,7 +16,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 import lombok.experimental.UtilityClass;
@@ -115,7 +115,7 @@ public class TableNodeFactory {
         if (cell.getTextBlocks().isEmpty()) {
             tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
         } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getWords(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else if (firstTextBlockIsHeadline(cell)) {
             SectionNodeFactory.addSection(layoutParsingType,
@@ -129,7 +129,7 @@ public class TableNodeFactory {
                                           context,
                                           document);
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<Word> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
+            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
             textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TextBlockFactory.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;
 
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
@@ -17,14 +17,14 @@ public class TextBlockFactory {
     long textBlockIdx;
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
 
         Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
         return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
     }
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
 
         SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionDto(sequences);
         int offset = stringOffset;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicExtractorService.java

@@ -11,7 +11,7 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 
 import lombok.RequiredArgsConstructor;
@@ -33,10 +33,10 @@ public class GraphicExtractorService {
                                                 PDPage pdPage,
                                                 int pageNumber,
                                                 CleanRulings cleanRulings,
-                                                List<Word> words,
+                                                List<TextPositionSequence> textPositionSequences,
                                                 boolean graphicsRaster) {
 
-        List<Box> characterBBoxes = getCharacterBBoxes(words);
+        List<Box> characterBBoxes = getCharacterBBoxes(textPositionSequences);
         List<Box> classifiedRulingsBoxes = getLineBBoxesOfAllClassifiedRulings(cleanRulings);
 
         GraphicBBDetector graphicBBDetector = new GraphicBBDetector(pdPage, true);
@@ -63,9 +63,9 @@ public class GraphicExtractorService {
     }
 
 
-    private List<Box> getCharacterBBoxes(List<Word> words) {
+    private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream()
+        return textPositionSequences.stream()
                 .map(BoundingBox::getBBoxPdf)
                 .map(Box::new)
                 .collect(Collectors.toList());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java

@@ -1,10 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
-import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
-import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;
-
 import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -13,15 +9,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
@@ -40,57 +32,44 @@ public class DocumentDataMapper {
     public DocumentData toDocumentData(Document document) {
 
         List<DocumentTextData> documentTextData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks()
-                        .stream())
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
                 .distinct()
                 .map(DocumentDataMapper::toAtomicTextBlockData)
                 .toList();
 
-        AllDocumentTextData allDocumentTextData = AllDocumentTextData.newBuilder().addAllDocumentTextData(documentTextData).build();
-
         List<DocumentPositionData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks()
-                        .stream())
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
                 .distinct()
                 .map(DocumentDataMapper::toAtomicPositionBlockData)
                 .toList();
 
-        AllDocumentPositionData allDocumentPositionData = AllDocumentPositionData.newBuilder().addAllDocumentPositionData(atomicPositionBlockData).build();
+        Set<Long> nonEmptyTextBlocks = documentTextData.stream().mapToLong(DocumentTextData::getId).boxed().collect(Collectors.toSet());
 
-        Set<Long> nonEmptyTextBlocks = documentTextData.stream()
-                .mapToLong(DocumentTextData::getId).boxed()
-                .collect(Collectors.toSet());
-
-        List<DocumentPage> documentPageData = document.getPages()
-                .stream()
-                .map(DocumentDataMapper::toPageData)
-                .toList();
-
-        AllDocumentPages allDocumentPages = AllDocumentPages.newBuilder().addAllDocumentPages(documentPageData).build();
-        DocumentStructureWrapper tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
+        List<DocumentPage> documentPageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList();
+        DocumentStructure tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
         return DocumentData.builder()
-                .documentTextData(allDocumentTextData)
-                .documentPositions(allDocumentPositionData)
-                .documentPages(allDocumentPages)
-                .documentStructureWrapper(tableOfContentsData)
+                .documentTextData(documentTextData.toArray(new DocumentTextData[0]))
+                .documentPositions(atomicPositionBlockData.toArray(new DocumentPositionData[0]))
+                .documentPages(documentPageData.toArray(new DocumentPage[0]))
+                .documentStructure(tableOfContentsData)
                 .build();
     }
 
 
-    private DocumentStructureWrapper toDocumentTreeData(DocumentTree documentTree) {
+    private DocumentStructure toDocumentTreeData(DocumentTree documentTree) {
 
-        return new DocumentStructureWrapper(DocumentStructure.newBuilder().setRoot(toEntryData(documentTree.getRoot())).build());
+        return new DocumentStructure(toEntryData(documentTree.getRoot()));
     }
 
 
-    private EntryData toEntryData(DocumentTree.Entry entry) {
+    private DocumentStructure.EntryData toEntryData(DocumentTree.Entry entry) {
 
-        List<Long> atomicTextBlocks;
+        Long[] atomicTextBlocks;
 
         if (entry.getNode().isLeaf()) {
             atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getLeafTextBlock());
         } else {
-            atomicTextBlocks = new ArrayList<>();
+            atomicTextBlocks = new Long[]{};
         }
 
         Map<String, String> properties = switch (entry.getType()) {
@@ -102,81 +81,77 @@ public class DocumentDataMapper {
             default -> new HashMap<>();
         };
 
-        var documentBuilder = EntryData.newBuilder()
-                .addAllTreeId(entry.getTreeId())
-                .addAllChildren(entry.getChildren()
-                                        .stream()
-                                        .map(DocumentDataMapper::toEntryData)
-                                        .toList())
-                .setType(entry.getType())
-                .addAllAtomicBlockIds(atomicTextBlocks)
-                .addAllPageNumbers(entry.getNode().getPages()
-                                           .stream()
-                                           .map(Page::getNumber)
-                                           .map(Integer::longValue)
-                                           .toList())
-                .putAllProperties(properties);
+        DocumentStructure.EntryData.EntryDataBuilder documentBuilder = DocumentStructure.EntryData.builder()
+                .treeId(toPrimitiveIntArray(entry.getTreeId()))
+                .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
+                .type(entry.getType())
+                .atomicBlockIds(atomicTextBlocks)
+                .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new))
+                .properties(properties);
         if (entry.getNode() != null) {
-            documentBuilder.addAllEngines(entry.getNode().getEngines());
+            documentBuilder.engines(entry.getNode().getEngines());
         } else {
-            documentBuilder.addAllEngines(new HashSet<>(Set.of(LayoutEngine.ALGORITHM)));
+            documentBuilder.engines(new HashSet<>(Set.of(LayoutEngine.ALGORITHM)));
         }
         return documentBuilder.build();
     }
 
 
-    private List<Long> toAtomicTextBlockIds(TextBlock textBlock) {
+    private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
 
-        return textBlock.getAtomicTextBlocks()
-                .stream()
-                .map(AtomicTextBlock::getId)
-                .toList();
+        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
     }
 
 
     private DocumentPage toPageData(Page p) {
 
-        return DocumentPage.newBuilder().setRotation(p.getRotation()).setHeight(p.getHeight()).setWidth(p.getWidth()).setNumber(p.getNumber()).build();
+        return DocumentPage.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
     }
 
 
     private DocumentTextData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
 
-        return DocumentTextData.newBuilder()
-                .setId(atomicTextBlock.getId())
-                .setPage(atomicTextBlock.getPage().getNumber().longValue())
-                .setSearchText(atomicTextBlock.getSearchText())
-                .setNumberOnPage(atomicTextBlock.getNumberOnPage())
-                .setStart(atomicTextBlock.getTextRange().start())
-                .setEnd(atomicTextBlock.getTextRange().end())
-                .addAllLineBreaks(atomicTextBlock.getLineBreaks())
+        return DocumentTextData.builder()
+                .id(atomicTextBlock.getId())
+                .page(atomicTextBlock.getPage().getNumber().longValue())
+                .searchText(atomicTextBlock.getSearchText())
+                .numberOnPage(atomicTextBlock.getNumberOnPage())
+                .start(atomicTextBlock.getTextRange().start())
+                .end(atomicTextBlock.getTextRange().end())
+                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
                 .build();
     }
 
 
     private DocumentPositionData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
 
-        return DocumentPositionData.newBuilder()
-                .setId(atomicTextBlock.getId())
-                .addAllPositions(toPositions(atomicTextBlock.getPositions()))
-                .addAllStringIdxToPositionIdx(atomicTextBlock.getStringIdxToPositionIdx())
+        return DocumentPositionData.builder()
+                .id(atomicTextBlock.getId())
+                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
+                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
                 .build();
     }
 
 
-    private static List<Position> toPositions(List<Rectangle2D> rects) {
+    private static float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
 
-        List<Position> positions = new ArrayList<>();
-        for (Rectangle2D rect : rects) {
-            positions.add(toPosition(rect));
+        float[][] positionMatrix = new float[positions.size()][];
+        for (int i = 0; i < positions.size(); i++) {
+            positionMatrix[i] = toArray(positions.get(i));
         }
-        return positions;
+        return positionMatrix;
     }
 
 
-    private static Position toPosition(Rectangle2D rect) {
+    private static float[] toArray(Rectangle2D positions) {
 
-        return Position.newBuilder().addValue((float) rect.getMinX()).addValue((float) rect.getMinY()).addValue((float) rect.getWidth()).addValue((float) rect.getHeight()).build();
+        return new float[]{(float) positions.getMinX(), (float) positions.getMinY(), (float) positions.getWidth(), (float) positions.getHeight()};
+    }
+
+
+    private int[] toPrimitiveIntArray(List<Integer> list) {
+
+        return list.stream().mapToInt(Integer::intValue).toArray();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentGraphMapper.java

@@ -8,10 +8,10 @@ import java.util.Map;
 import java.util.NoSuchElementException;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
@@ -41,29 +41,27 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pages.addAll(documentData.getDocumentPages().getDocumentPagesList()
-                                     .stream()
+        context.pages.addAll(Arrays.stream(documentData.getDocumentPages())
                                      .map(DocumentGraphMapper::buildPage)
                                      .toList());
 
-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildrenList(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
 
         document.setDocumentTree(context.documentTree);
         document.setPages(new HashSet<>(context.pages));
-        document.setNumberOfPages(documentData.getDocumentPages().getDocumentPagesCount());
+        document.setNumberOfPages(documentData.getDocumentPages().length);
 
         document.setTextBlock(document.getTextBlock());
         return document;
     }
 
 
-    private List<DocumentTree.Entry> buildEntries(List<EntryData> entries, Context context) {
+    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
 
         List<DocumentTree.Entry> newEntries = new LinkedList<>();
-        for (EntryData entryData : entries) {
+        for (DocumentStructure.EntryData entryData : entries) {
 
-            List<Page> pages = entryData.getPageNumbersList()
-                    .stream()
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
                     .map(pageNumber -> getPage(pageNumber, context))
                     .toList();
 
@@ -76,12 +74,12 @@ public class DocumentGraphMapper {
                 case FOOTER -> buildFooter(context);
                 case TABLE -> buildTable(context, entryData.getProperties());
                 case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbersList());
+                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
                 default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
             };
 
-            if (entryData.getAtomicBlockIdsCount() > 0) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIdsList(), context, node);
+            if (entryData.getAtomicBlockIds().length > 0) {
+                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
                 switch (entryData.getType()) {
                     case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
@@ -91,10 +89,11 @@ public class DocumentGraphMapper {
                             .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
                 }
             }
-            List<Integer> treeId = entryData.getTreeIdList();
+            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
+                    .toList();
             node.setTreeId(treeId);
 
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
+            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
         }
         return newEntries;
     }
@@ -106,10 +105,10 @@ public class DocumentGraphMapper {
     }
 
 
-    private Image buildImage(Context context, Map<String, String> properties, List<Long> pageNumbers) {
+    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
 
-        assert pageNumbers.size() == 1;
-        Page page = getPage(pageNumbers.get(0), context);
+        assert pageNumbers.length == 1;
+        Page page = getPage(pageNumbers[0], context);
         var builder = Image.builder();
         PropertiesMapper.parseImageProperties(properties, builder);
         return builder.documentTree(context.documentTree).page(page).build();
@@ -162,7 +161,7 @@ public class DocumentGraphMapper {
 
             DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
 
-            var unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
             duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
             return duplicatedParagraph;
 
@@ -172,9 +171,9 @@ public class DocumentGraphMapper {
     }
 
 
-    private TextBlock toTextBlock(List<Long> atomicTextBlockIds, Context context, SemanticNode parent) {
+    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return atomicTextBlockIds.stream()
+        return Arrays.stream(atomicTextBlockIds)
                 .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
                 .collect(new TextBlockCollector());
     }
@@ -182,13 +181,10 @@ public class DocumentGraphMapper {
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
-        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.getDocumentTextDataList()
-                                                               .get(Math.toIntExact(atomicTextBlockId)),
-                                                       context.atomicPositionBlockData.getDocumentPositionDataList()
-                                                               .get(Math.toIntExact(atomicTextBlockId)),
+        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)),
+                                                       context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
                                                        parent,
-                                                       getPage(context.documentTextDataBlockData.getDocumentTextDataList()
-                                                                       .get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
     }
 
 
@@ -211,16 +207,18 @@ public class DocumentGraphMapper {
 
         private final DocumentTree documentTree;
         private final List<Page> pages;
-        private final AllDocumentTextData documentTextDataBlockData;
-        private final AllDocumentPositionData atomicPositionBlockData;
+        private final List<DocumentTextData> documentTextDataBlockData;
+        private final List<DocumentPositionData> atomicPositionBlockData;
 
 
         Context(DocumentData documentData, DocumentTree documentTree) {
 
             this.documentTree = documentTree;
             this.pages = new LinkedList<>();
-            this.documentTextDataBlockData = documentData.getDocumentTextData();
-            this.atomicPositionBlockData = documentData.getDocumentPositions();
+            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData())
+                    .toList();
+            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions())
+                    .toList();
 
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -3,11 +3,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 import java.awt.geom.Rectangle2D;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
@@ -21,11 +20,11 @@ public class PropertiesMapper {
     public static Map<String, String> buildImageProperties(Image image) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
-        properties.put(DocumentStructureWrapper.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
-        properties.put(DocumentStructureWrapper.ImageProperties.POSITION, toString(image.getPosition()));
-        properties.put(DocumentStructureWrapper.ImageProperties.ID, image.getId());
-        properties.put(DocumentStructureWrapper.ImageProperties.REPRESENTATION_HASH, image.getRepresentationHash());
+        properties.put(DocumentStructure.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
+        properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
+        properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.ID, image.getId());
+        properties.put(DocumentStructure.ImageProperties.REPRESENTATION_HASH, image.getRepresentationHash());
         return properties;
     }
 
@@ -33,19 +32,15 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(DocumentStructureWrapper.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
-        properties.put(DocumentStructureWrapper.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
-        properties.put(DocumentStructureWrapper.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
+        properties.put(DocumentStructure.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
+        properties.put(DocumentStructure.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
+        properties.put(DocumentStructure.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
 
         if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
             throw new IllegalArgumentException("TableCell can only occur on a single page!");
         }
-        String bBoxString = toString(tableCell.getBBox()
-                                             .get(tableCell.getPages()
-                                                          .stream()
-                                                          .findFirst()
-                                                          .get()));
-        properties.put(DocumentStructureWrapper.TableCellProperties.B_BOX, bBoxString);
+        String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
+        properties.put(DocumentStructure.TableCellProperties.B_BOX, bBoxString);
 
         return properties;
     }
@@ -54,62 +49,59 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableProperties(Table table) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
-        properties.put(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
         return properties;
     }
 
 
     public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
 
-        builder.imageType(parseImageType(properties.get(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.ImageProperties.TRANSPARENT)));
-        builder.position(DocumentStructureWrapper.parseRectangle2D(properties.get(DocumentStructureWrapper.ImageProperties.POSITION)));
+        builder.imageType(parseImageType(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
+        builder.position(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
     }
 
 
     public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
 
-        builder.row(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.ROW)));
-        builder.col(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.COL)));
-        builder.header(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.TableCellProperties.HEADER)));
-        builder.bBox(DocumentStructureWrapper.parseRectangle2D(properties.get(DocumentStructureWrapper.TableCellProperties.B_BOX)));
+        builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
+        builder.bBox(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
     }
 
 
     public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
 
-        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS)));
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
     }
 
 
     public static Map<String, String> buildDuplicateParagraphProperties(DuplicatedParagraph duplicatedParagraph) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID,
-                       Arrays.toString(toAtomicTextBlockIds(duplicatedParagraph.getUnsortedLeafTextBlock())));
+        properties.put(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID, Arrays.toString(toAtomicTextBlockIds(duplicatedParagraph.getUnsortedLeafTextBlock())));
         return properties;
     }
 
 
     public static boolean isDuplicateParagraph(Map<String, String> properties) {
 
-        return properties.containsKey(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+        return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
     }
 
 
-    public static List<Long> getUnsortedTextblockIds(Map<String, String> properties) {
+    public static Long[] getUnsortedTextblockIds(Map<String, String> properties) {
 
-        return toLongList(properties.get(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+        return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
     }
 
 
-    public static List<Long> toLongList(String ids) {
+    public static Long[] toLongArray(String ids) {
 
-        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(","))
-                .map(Long::valueOf)
-                .toList();
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new);
     }
 
 
@@ -128,23 +120,20 @@ public class PropertiesMapper {
     public static String toString(Rectangle2D rectangle2D) {
 
         return String.format(Locale.US,
-                             "%f%s%f%s%f%s%f",
-                             rectangle2D.getX(),
-                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
-                             rectangle2D.getY(),
-                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
-                             rectangle2D.getWidth(),
-                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
-                             rectangle2D.getHeight());
+                "%f%s%f%s%f%s%f",
+                rectangle2D.getX(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getY(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getWidth(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getHeight());
     }
 
 
     private static Long[] toAtomicTextBlockIds(TextBlock textBlock) {
 
-        return textBlock.getAtomicTextBlocks()
-                .stream()
-                .map(AtomicTextBlock::getId)
-                .toArray(Long[]::new);
+        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/TaasDocumentDataMapper.java

@@ -6,7 +6,7 @@ import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.IntStream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ParagraphData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Range;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFLinesTextStripper.java

@@ -5,7 +5,6 @@ import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
@@ -40,7 +39,7 @@ import org.apache.pdfbox.text.TextPosition;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Getter;
 import lombok.Setter;
@@ -51,8 +50,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {
 
-    private final static Set<String> DOT_LIKE_CHARACTERS = Set.of(".", "·", "•", "․", "‧", "∙", "⋅", "・", "．", "･", "…", "⸱", "﹒", "ꞏ");
-    private final List<Word> words = new ArrayList<>();
+    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
     private final List<Ruling> rulings = new ArrayList<>();
     private final List<Ruling> graphicsPath = new ArrayList<>();
     @Setter
@@ -246,10 +244,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 direction = textPositions.get(i).getDir();
             }
 
-            if (!words.isEmpty()) {
-                previous = words.get(words.size() - 1)
+            if (!textPositionSequences.isEmpty()) {
+                previous = textPositionSequences.get(textPositionSequences.size() - 1)
                         .getTextPositions()
-                        .get(words.get(words.size() - 1).getTextPositions().size() - 1);
+                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
             }
 
             if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
@@ -259,7 +257,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
             if (textPositions.get(i).getDir() != direction && startIndex != i) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 startIndex = i;
                 direction = textPositions.get(i).getDir();
             }
@@ -268,7 +266,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
             if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
                 if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
             }
@@ -276,7 +274,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
             if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
                 if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
             }
@@ -290,22 +288,22 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                     // Remove false sequence ends (whitespaces)
                     if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                         for (TextPosition t : sublist) {
-                            words.get(words.size() - 1).add(t);
+                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                         }
                     } else {
-                        words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                     }
                 }
                 startIndex = i + 1;
             }
             if (isDottedLineFollowedByWord(textPositions, i, startIndex)) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 startIndex = i;
             }
             if (isWordFollowedByDottedLine(textPositions, i, startIndex)) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i - 2);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 startIndex = i - 2;
             }
         }
@@ -324,10 +322,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
                 && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                 for (TextPosition t : sublist) {
-                    words.get(words.size() - 1).add(t);
+                    textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                 }
             } else {
-                words.add(new Word(sublist, pageNumber, isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
             }
         }
 
@@ -338,32 +336,20 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     private boolean isWordFollowedByDottedLine(List<TextPosition> textPositions, int i, int startIndex) {
 
         return i - startIndex >= 4 //
-               && isDot(textPositions, i) //
-               && isDot(textPositions, i - 1) //
-               && isDot(textPositions, i - 2) //
-               && alphanumeric(textPositions, i - 3);
+               && textPositions.get(i).getUnicode().equals(".") //
+               && textPositions.get(i - 1).getUnicode().equals(".") //
+               && textPositions.get(i - 2).getUnicode().equals(".") //
+               && !textPositions.get(i - 3).getUnicode().equals(".");
     }
 
 
     private static boolean isDottedLineFollowedByWord(List<TextPosition> textPositions, int i, int startIndex) {
 
         return i - startIndex >= 4 //
-               && alphanumeric(textPositions, i)  //
-               && isDot(textPositions, i - 1) //
-               && isDot(textPositions, i - 2) //
-               && isDot(textPositions, i - 3);
-    }
-
-
-    private static boolean isDot(List<TextPosition> textPositions, int i) {
-
-        return DOT_LIKE_CHARACTERS.contains(textPositions.get(i).getUnicode());
-    }
-
-
-    private static boolean alphanumeric(List<TextPosition> textPositions, int i) {
-
-        return Character.isAlphabetic(textPositions.get(i).getUnicode().charAt(0)) || Character.isDigit(textPositions.get(i).getUnicode().charAt(0));
+               && !textPositions.get(i).getUnicode().equals(".") //
+               && textPositions.get(i - 1).getUnicode().equals(".") //
+               && textPositions.get(i - 2).getUnicode().equals(".") //
+               && textPositions.get(i - 3).getUnicode().equals(".");
     }
 
 
@@ -392,7 +378,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     @Override
     public String getText(PDDocument doc) throws IOException {
 
-        words.clear();
+        textPositionSequences.clear();
         rulings.clear();
         graphicsPath.clear();
         path_x = 0.0f;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/MarkedContentUtils.java

@@ -13,7 +13,7 @@ import org.apache.pdfbox.text.TextPosition;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -48,7 +48,7 @@ public class MarkedContentUtils {
 
         return markedContentByYPosition.values()
                 .stream()
-                .map(textPositions -> new Word(textPositions, 0, true).getBBoxPdf())
+                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxPdf())
                 .map(t -> new Rectangle2D.Double(t.getX(), t.getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
                 .collect(Collectors.toList());
     }
@@ -89,7 +89,7 @@ public class MarkedContentUtils {
                     .filter(content -> content instanceof TextPosition)
                     .map(content -> (TextPosition) content)
                     .filter(content -> !content.getUnicode().equals(" "))
-                    .map(textPositions -> new Word(List.of(textPositions), 0, true))
+                    .map(textPositions -> new TextPositionSequence(List.of(textPositions), 0, true))
                     .map(BoundingBox::getBBoxPdf)
                     .collect(Collectors.toList());
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PdfVisualisationUtility.java

@@ -14,7 +14,7 @@ import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
 import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
@@ -117,7 +117,6 @@ public class PdfVisualisationUtility {
             case TABLE -> Color.ORANGE;
             case TABLE_CELL -> Color.GRAY;
             case IMAGE -> Color.MAGENTA;
-            case UNRECOGNIZED -> Color.PINK;
         }).build();
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/ProtobufUtil.java

@@ -1,42 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.utils;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.zip.GZIPOutputStream;
-
-import com.google.protobuf.Message;
-import com.google.protobuf.MessageOrBuilder;
-import com.google.protobuf.Struct;
-import com.google.protobuf.util.JsonFormat;
-
-import lombok.SneakyThrows;
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class ProtobufUtil {
-
-    public static String toJson(MessageOrBuilder messageOrBuilder) throws IOException {
-        return JsonFormat.printer().print(messageOrBuilder);
-    }
-
-    @SuppressWarnings("unchecked")
-    public static Message fromJson(String json) throws IOException {
-        Message.Builder structBuilder = Struct.newBuilder();
-        JsonFormat.parser().ignoringUnknownFields().merge(json, structBuilder);
-        return structBuilder.build();
-    }
-
-
-    @SneakyThrows
-    public <T extends Message> File serializeToTempFile(T any) {
-        var tempFile = File.createTempFile("storage-protobuf", ".data");
-
-        try (var fos = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(tempFile)))) {
-            any.writeTo(fos);
-            return tempFile;
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionOperations.java

@@ -16,7 +16,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Union
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -36,33 +36,33 @@ public class TextPositionOperations {
                     .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, X_THRESHOLD));
 
 
-    public List<Word> mergeAndSort(List<TextPageBlock> textBlocks) {
+    public List<TextPositionSequence> mergeAndSort(List<TextPageBlock> textBlocks) {
 
         var sequences = textBlocks.stream()
-                .flatMap(tb -> tb.getWords()
+                .flatMap(tb -> tb.getSequences()
                         .stream())
                 .collect(Collectors.toSet());
         return sortUsingLineDetection(sequences);
     }
 
 
-    public List<Word> sort(List<Word> sequences) {
+    public List<TextPositionSequence> sort(List<TextPositionSequence> sequences) {
 
         return sortUsingLineDetection(new HashSet<>(sequences));
     }
 
 
-    private List<Word> sortUsingLineDetection(Set<Word> sequences) {
+    private List<TextPositionSequence> sortUsingLineDetection(Set<TextPositionSequence> sequences) {
 
         return sortLines(groupByLine(sequences));
     }
 
 
-    public List<Word> sortLines(Collection<Set<Word>> lines) {
+    public List<TextPositionSequence> sortLines(Collection<Set<TextPositionSequence>> lines) {
 
-        List<List<Word>> lineBlocks = new ArrayList<>();
-        for (Set<Word> line : lines) {
-            List<Word> sortedLine = sortByXDirAdj(line);
+        List<List<TextPositionSequence>> lineBlocks = new ArrayList<>();
+        for (Set<TextPositionSequence> line : lines) {
+            List<TextPositionSequence> sortedLine = sortByXDirAdj(line);
             if (!sortedLine.isEmpty()) {
                 lineBlocks.add(sortedLine);
             }
@@ -70,35 +70,35 @@ public class TextPositionOperations {
         // need to use old sorting, since COMPARATOR_DIR_ADJ is not transitive
         QuickSort.sort(lineBlocks, Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ));
 
-        List<Word> list = new ArrayList<>();
-        for (List<Word> words : lineBlocks) {
-            list.addAll(words);
+        List<TextPositionSequence> list = new ArrayList<>();
+        for (List<TextPositionSequence> textPositionSequences : lineBlocks) {
+            list.addAll(textPositionSequences);
         }
         return list;
     }
 
 
-    private List<Word> sortByXDirAdj(Set<Word> line) {
+    private List<TextPositionSequence> sortByXDirAdj(Set<TextPositionSequence> line) {
 
         return line.stream()
-                .sorted(Comparator.comparing(Word::getXDirAdj))
+                .sorted(Comparator.comparing(TextPositionSequence::getXDirAdj))
                 .toList();
     }
 
 
-    public Collection<Set<Word>> groupByLine(Set<Word> sequences) {
+    public Collection<Set<TextPositionSequence>> groupByLine(Set<TextPositionSequence> sequences) {
 
         double maxLineDistance = sequences.stream()
-                                         .map(Word::getBBoxDirAdj)
+                                         .map(TextPositionSequence::getBBoxDirAdj)
                                          .mapToDouble(RectangularShape::getHeight).average().orElse(10) * MAX_LINE_HEIGHT_FACTOR;
         double maxXGap = sequences.stream()
-                                 .map(Word::getBBoxDirAdj)
+                                 .map(TextPositionSequence::getBBoxDirAdj)
                                  .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;
 
-        UnionFind<Word> unionFind = new UnionFind<>(sequences);
+        UnionFind<TextPositionSequence> unionFind = new UnionFind<>(sequences);
 
-        for (Word sequence : sequences) {
-            for (Word sequence2 : sequences) {
+        for (TextPositionSequence sequence : sequences) {
+            for (TextPositionSequence sequence2 : sequences) {
 
                 if (sequence.equals(sequence2)) { // || unionFind.inSameSet(sequence, sequence2)) doing this is actually slower than not doing it
                     continue;
@@ -144,10 +144,10 @@ public class TextPositionOperations {
     }
 
 
-    public List<Word> merge(List<TextPageBlock> textBlocks) {
+    public List<TextPositionSequence> merge(List<TextPageBlock> textBlocks) {
 
         return textBlocks.stream()
-                .map(TextPageBlock::getWords)
+                .map(TextPageBlock::getSequences)
                 .flatMap(Collection::stream)
                 .collect(Collectors.toList());
     }