RED-10380: wip add table of contents node type

RED-10247: dictionary entry not found in footer due to wrong text sorting

See merge request fforesight/layout-parser!251

.gitlab-ci.yml

@@ -21,5 +21,6 @@ deploy:
       dotenv: version.env
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^feature/ && $CI_COMMIT_TAG == ""
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -42,6 +42,19 @@ tasks.jacocoTestReport {
 }
 
 allprojects {
+
+    tasks.withType<Javadoc> {
+        options {
+            this as StandardJavadocDocletOptions
+            addBooleanOption("Xdoclint:none", true)
+            addStringOption("Xmaxwarns", "1")
+        }
+    }
+
+    pmd {
+        setConsoleOutput(true)
+    }
+
     publishing {
         publications {
             create<MavenPublication>(name) {
@@ -64,6 +77,7 @@ java {
     withJavadocJar()
 }
 
+
 repositories {
     mavenLocal()
     mavenCentral()

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -7,4 +7,5 @@ description = "layoutparser-service-internal-api"
 
 dependencies {
     implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
+    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -2,6 +2,11 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.io.Serializable;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
+
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -13,16 +18,26 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: " + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. " + "The text, which is stored as separate blocks of data. " + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. " + "The page information.")
+@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: "
+                      + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. "
+                      + "The text, which is stored as separate blocks of data. "
+                      + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. "
+                      + "The page information.")
 public class DocumentData implements Serializable {
 
     @Schema(description = "Contains information about the document's pages.")
-    DocumentPage[] documentPages;
+    AllDocumentPages documentPages;
     @Schema(description = "Contains information about the document's text.")
-    DocumentTextData[] documentTextData;
+    AllDocumentTextData documentTextData;
     @Schema(description = "Contains information about the document's text positions.")
-    DocumentPositionData[] documentPositions;
+    AllDocumentPositionData documentPositions;
     @Schema(description = "Contains information about the document's semantic structure.")
-    DocumentStructure documentStructure;
+    DocumentStructureWrapper documentStructureWrapper;
+
+
+    public DocumentStructure getDocumentStructure() {
+
+        return documentStructureWrapper.getDocumentStructure();
+    }
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -10,6 +10,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -10,6 +10,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -16,6 +16,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureProto.java

@@ -0,0 +1,799 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: DocumentStructure.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class DocumentStructureProto {
+
+    private DocumentStructureProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", DocumentStructureProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    public interface DocumentStructureOrBuilder extends
+            // @@protoc_insertion_point(interface_extends:DocumentStructure)
+            com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return Whether the root field is set.
+         */
+        boolean hasRoot();
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return The root.
+         */
+        EntryDataProto.EntryData getRoot();
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         */
+        EntryDataProto.EntryDataOrBuilder getRootOrBuilder();
+
+    }
+
+    /**
+     * Protobuf type {@code DocumentStructure}
+     */
+    public static final class DocumentStructure extends com.google.protobuf.GeneratedMessage implements
+            // @@protoc_insertion_point(message_implements:DocumentStructure)
+            DocumentStructureOrBuilder {
+
+        private static final long serialVersionUID = 0L;
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", DocumentStructure.class.getName());
+        }
+
+        // Use DocumentStructure.newBuilder() to construct.
+        private DocumentStructure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+
+            super(builder);
+        }
+
+
+        private DocumentStructure() {
+
+        }
+
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+
+            return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+        }
+
+
+        @Override
+        protected FieldAccessorTable internalGetFieldAccessorTable() {
+
+            return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
+        }
+
+
+        private int bitField0_;
+        public static final int ROOT_FIELD_NUMBER = 1;
+        private EntryDataProto.EntryData root_;
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return Whether the root field is set.
+         */
+        @Override
+        public boolean hasRoot() {
+
+            return ((bitField0_ & 0x00000001) != 0);
+        }
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return The root.
+         */
+        @Override
+        public EntryDataProto.EntryData getRoot() {
+
+            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+        }
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         */
+        @Override
+        public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+
+            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+        }
+
+
+        private byte memoizedIsInitialized = -1;
+
+
+        @Override
+        public final boolean isInitialized() {
+
+            byte isInitialized = memoizedIsInitialized;
+          if (isInitialized == 1) {
+            return true;
+          }
+          if (isInitialized == 0) {
+            return false;
+          }
+
+            memoizedIsInitialized = 1;
+            return true;
+        }
+
+
+        @Override
+        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+
+            if (((bitField0_ & 0x00000001) != 0)) {
+                output.writeMessage(1, getRoot());
+            }
+            getUnknownFields().writeTo(output);
+        }
+
+
+        @Override
+        public int getSerializedSize() {
+
+            int size = memoizedSize;
+          if (size != -1) {
+            return size;
+          }
+
+            size = 0;
+            if (((bitField0_ & 0x00000001) != 0)) {
+                size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, getRoot());
+            }
+            size += getUnknownFields().getSerializedSize();
+            memoizedSize = size;
+            return size;
+        }
+
+
+        @Override
+        public boolean equals(final Object obj) {
+
+            if (obj == this) {
+                return true;
+            }
+            if (!(obj instanceof DocumentStructure)) {
+                return super.equals(obj);
+            }
+            DocumentStructure other = (DocumentStructure) obj;
+
+          if (hasRoot() != other.hasRoot()) {
+            return false;
+          }
+            if (hasRoot()) {
+              if (!getRoot().equals(other.getRoot())) {
+                return false;
+              }
+            }
+          if (!getUnknownFields().equals(other.getUnknownFields())) {
+            return false;
+          }
+            return true;
+        }
+
+
+        @Override
+        public int hashCode() {
+
+            if (memoizedHashCode != 0) {
+                return memoizedHashCode;
+            }
+            int hash = 41;
+            hash = (19 * hash) + getDescriptor().hashCode();
+            if (hasRoot()) {
+                hash = (37 * hash) + ROOT_FIELD_NUMBER;
+                hash = (53 * hash) + getRoot().hashCode();
+            }
+            hash = (29 * hash) + getUnknownFields().hashCode();
+            memoizedHashCode = hash;
+            return hash;
+        }
+
+
+        public static DocumentStructure parseFrom(java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(java.nio.ByteBuffer data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(byte[] data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(java.io.InputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        @Override
+        public Builder newBuilderForType() {return newBuilder();}
+
+
+        public static Builder newBuilder() {
+
+            return DEFAULT_INSTANCE.toBuilder();
+        }
+
+
+        public static Builder newBuilder(DocumentStructure prototype) {
+
+            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+
+        @Override
+        public Builder toBuilder() {
+
+            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+
+        @Override
+        protected Builder newBuilderForType(BuilderParent parent) {
+
+            Builder builder = new Builder(parent);
+            return builder;
+        }
+
+
+        /**
+         * Protobuf type {@code DocumentStructure}
+         */
+        public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> implements
+                // @@protoc_insertion_point(builder_implements:DocumentStructure)
+                DocumentStructureOrBuilder {
+
+            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+            }
+
+
+            @Override
+            protected FieldAccessorTable internalGetFieldAccessorTable() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
+            }
+
+
+            // Construct using DocumentStructureOuterClass.DocumentStructure.newBuilder()
+            private Builder() {
+
+                maybeForceBuilderInitialization();
+            }
+
+
+            private Builder(BuilderParent parent) {
+
+                super(parent);
+                maybeForceBuilderInitialization();
+            }
+
+
+            private void maybeForceBuilderInitialization() {
+
+                if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+                    getRootFieldBuilder();
+                }
+            }
+
+
+            @Override
+            public Builder clear() {
+
+                super.clear();
+                bitField0_ = 0;
+                root_ = null;
+                if (rootBuilder_ != null) {
+                    rootBuilder_.dispose();
+                    rootBuilder_ = null;
+                }
+                return this;
+            }
+
+
+            @Override
+            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+            }
+
+
+            @Override
+            public DocumentStructure getDefaultInstanceForType() {
+
+                return DocumentStructure.getDefaultInstance();
+            }
+
+
+            @Override
+            public DocumentStructure build() {
+
+                DocumentStructure result = buildPartial();
+                if (!result.isInitialized()) {
+                    throw newUninitializedMessageException(result);
+                }
+                return result;
+            }
+
+
+            @Override
+            public DocumentStructure buildPartial() {
+
+                DocumentStructure result = new DocumentStructure(this);
+                if (bitField0_ != 0) {
+                    buildPartial0(result);
+                }
+                onBuilt();
+                return result;
+            }
+
+
+            private void buildPartial0(DocumentStructure result) {
+
+                int from_bitField0_ = bitField0_;
+                int to_bitField0_ = 0;
+                if (((from_bitField0_ & 0x00000001) != 0)) {
+                    result.root_ = rootBuilder_ == null ? root_ : rootBuilder_.build();
+                    to_bitField0_ |= 0x00000001;
+                }
+                result.bitField0_ |= to_bitField0_;
+            }
+
+
+            @Override
+            public Builder mergeFrom(com.google.protobuf.Message other) {
+
+                if (other instanceof DocumentStructure) {
+                    return mergeFrom((DocumentStructure) other);
+                } else {
+                    super.mergeFrom(other);
+                    return this;
+                }
+            }
+
+
+            public Builder mergeFrom(DocumentStructure other) {
+
+              if (other == DocumentStructure.getDefaultInstance()) {
+                return this;
+              }
+                if (other.hasRoot()) {
+                    mergeRoot(other.getRoot());
+                }
+                this.mergeUnknownFields(other.getUnknownFields());
+                onChanged();
+                return this;
+            }
+
+
+            @Override
+            public final boolean isInitialized() {
+
+                return true;
+            }
+
+
+            @Override
+            public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+                if (extensionRegistry == null) {
+                    throw new NullPointerException();
+                }
+                try {
+                    boolean done = false;
+                    while (!done) {
+                        int tag = input.readTag();
+                        switch (tag) {
+                            case 0:
+                                done = true;
+                                break;
+                            case 10: {
+                                input.readMessage(getRootFieldBuilder().getBuilder(), extensionRegistry);
+                                bitField0_ |= 0x00000001;
+                                break;
+                            } // case 10
+                            default: {
+                                if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                                    done = true; // was an endgroup tag
+                                }
+                                break;
+                            } // default:
+                        } // switch (tag)
+                    } // while (!done)
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    throw e.unwrapIOException();
+                } finally {
+                    onChanged();
+                } // finally
+                return this;
+            }
+
+
+            private int bitField0_;
+
+            private EntryDataProto.EntryData root_;
+            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> rootBuilder_;
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             *
+             * @return Whether the root field is set.
+             */
+            public boolean hasRoot() {
+
+                return ((bitField0_ & 0x00000001) != 0);
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             *
+             * @return The root.
+             */
+            public EntryDataProto.EntryData getRoot() {
+
+                if (rootBuilder_ == null) {
+                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+                } else {
+                    return rootBuilder_.getMessage();
+                }
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder setRoot(EntryDataProto.EntryData value) {
+
+                if (rootBuilder_ == null) {
+                    if (value == null) {
+                        throw new NullPointerException();
+                    }
+                    root_ = value;
+                } else {
+                    rootBuilder_.setMessage(value);
+                }
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder setRoot(EntryDataProto.EntryData.Builder builderForValue) {
+
+                if (rootBuilder_ == null) {
+                    root_ = builderForValue.build();
+                } else {
+                    rootBuilder_.setMessage(builderForValue.build());
+                }
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder mergeRoot(EntryDataProto.EntryData value) {
+
+                if (rootBuilder_ == null) {
+                    if (((bitField0_ & 0x00000001) != 0) && root_ != null && root_ != EntryDataProto.EntryData.getDefaultInstance()) {
+                        getRootBuilder().mergeFrom(value);
+                    } else {
+                        root_ = value;
+                    }
+                } else {
+                    rootBuilder_.mergeFrom(value);
+                }
+                if (root_ != null) {
+                    bitField0_ |= 0x00000001;
+                    onChanged();
+                }
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder clearRoot() {
+
+                bitField0_ = (bitField0_ & ~0x00000001);
+                root_ = null;
+                if (rootBuilder_ != null) {
+                    rootBuilder_.dispose();
+                    rootBuilder_ = null;
+                }
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public EntryDataProto.EntryData.Builder getRootBuilder() {
+
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return getRootFieldBuilder().getBuilder();
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+
+                if (rootBuilder_ != null) {
+                    return rootBuilder_.getMessageOrBuilder();
+                } else {
+                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+                }
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> getRootFieldBuilder() {
+
+                if (rootBuilder_ == null) {
+                    rootBuilder_ = new com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder>(getRoot(),
+                                                                                                                                                                             getParentForChildren(),
+                                                                                                                                                                             isClean());
+                    root_ = null;
+                }
+                return rootBuilder_;
+            }
+
+            // @@protoc_insertion_point(builder_scope:DocumentStructure)
+        }
+
+        // @@protoc_insertion_point(class_scope:DocumentStructure)
+        private static final DocumentStructure DEFAULT_INSTANCE;
+
+        static {
+            DEFAULT_INSTANCE = new DocumentStructure();
+        }
+
+        public static DocumentStructure getDefaultInstance() {
+
+            return DEFAULT_INSTANCE;
+        }
+
+
+        private static final com.google.protobuf.Parser<DocumentStructure> PARSER = new com.google.protobuf.AbstractParser<DocumentStructure>() {
+            @Override
+            public DocumentStructure parsePartialFrom(com.google.protobuf.CodedInputStream input,
+                                                      com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+                Builder builder = newBuilder();
+                try {
+                    builder.mergeFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    throw e.setUnfinishedMessage(builder.buildPartial());
+                } catch (com.google.protobuf.UninitializedMessageException e) {
+                    throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial());
+                } catch (java.io.IOException e) {
+                    throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(builder.buildPartial());
+                }
+                return builder.buildPartial();
+            }
+        };
+
+
+        public static com.google.protobuf.Parser<DocumentStructure> parser() {
+
+            return PARSER;
+        }
+
+
+        @Override
+        public com.google.protobuf.Parser<DocumentStructure> getParserForType() {
+
+            return PARSER;
+        }
+
+
+        @Override
+        public DocumentStructure getDefaultInstanceForType() {
+
+            return DEFAULT_INSTANCE;
+        }
+
+    }
+
+    private static final com.google.protobuf.Descriptors.Descriptor internal_static_DocumentStructure_descriptor;
+    private static final com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_DocumentStructure_fieldAccessorTable;
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\027DocumentStructure.proto\032\017EntryData.pro"
+                                   + "to\"-\n\021DocumentStructure\022\030\n\004root\030\001 \001(\0132\n."
+                                   + "EntryDatab\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData,
+                                                                                                   new com.google.protobuf.Descriptors.FileDescriptor[]{EntryDataProto.getDescriptor(),});
+        internal_static_DocumentStructure_descriptor = getDescriptor().getMessageTypes()
+                .get(0);
+        internal_static_DocumentStructure_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(internal_static_DocumentStructure_descriptor,
+                                                                                                                           new String[]{"Root",});
+        descriptor.resolveAllFeaturesImmutable();
+        EntryDataProto.getDescriptor();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureWrapper.java

@@ -0,0 +1,126 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+
+import java.awt.geom.Rectangle2D;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@Getter
+@AllArgsConstructor
+public class DocumentStructureWrapper implements Serializable {
+
+    private final DocumentStructure documentStructure;
+
+
+    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+        public static final String REPRESENTATION_HASH = "representationHash";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a duplicate paragraph has in its properties field.")
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public static double[] parseRepresentationVector(String representationHash) {
+
+        String[] stringArray = representationHash.split("[,\\s]+");
+        double[] doubleArray = new double[stringArray.length];
+        for (int i = 0; i < stringArray.length; i++) {
+            doubleArray[i] = Double.parseDouble(stringArray[i]);
+        }
+
+        return doubleArray;
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return documentStructure.getRoot();
+        }
+        EntryData entry = documentStructure.getRoot().getChildrenList()
+                .get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.getChildrenList()
+                    .get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return Stream.concat(Stream.of(documentStructure.getRoot()),
+                             documentStructure.getRoot().getChildrenList()
+                                     .stream())
+                .flatMap(DocumentStructureWrapper::flatten);
+    }
+
+
+    public String toString() {
+
+        return String.join("\n",
+                           streamAllEntries().map(EntryData::toString)
+                                   .toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry),
+                             entry.getChildrenList()
+                                     .stream()
+                                     .flatMap(DocumentStructureWrapper::flatten));
+    }
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -10,6 +10,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
 @NoArgsConstructor

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+@Deprecated
 public enum LayoutEngine {
     ALGORITHM,
     AI,

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngineProto.java

@@ -0,0 +1,193 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: LayoutEngine.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class LayoutEngineProto {
+
+    private LayoutEngineProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", LayoutEngineProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    /**
+     * Protobuf enum {@code LayoutEngine}
+     */
+    public enum LayoutEngine implements com.google.protobuf.ProtocolMessageEnum {
+        /**
+         * <code>ALGORITHM = 0;</code>
+         */
+        ALGORITHM(0),
+        /**
+         * <code>AI = 1;</code>
+         */
+        AI(1),
+        /**
+         * <code>OUTLINE = 2;</code>
+         */
+        OUTLINE(2),
+        UNRECOGNIZED(-1),
+        ;
+
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", LayoutEngine.class.getName());
+        }
+
+        /**
+         * <code>ALGORITHM = 0;</code>
+         */
+        public static final int ALGORITHM_VALUE = 0;
+        /**
+         * <code>AI = 1;</code>
+         */
+        public static final int AI_VALUE = 1;
+        /**
+         * <code>OUTLINE = 2;</code>
+         */
+        public static final int OUTLINE_VALUE = 2;
+
+
+        public final int getNumber() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
+            }
+            return value;
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         * @deprecated Use {@link #forNumber(int)} instead.
+         */
+        @Deprecated
+        public static LayoutEngine valueOf(int value) {
+
+            return forNumber(value);
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         */
+        public static LayoutEngine forNumber(int value) {
+
+            switch (value) {
+                case 0:
+                    return ALGORITHM;
+                case 1:
+                    return AI;
+                case 2:
+                    return OUTLINE;
+                default:
+                    return null;
+            }
+        }
+
+
+        public static com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalGetValueMap() {
+
+            return internalValueMap;
+        }
+
+
+        private static final com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<LayoutEngine>() {
+            public LayoutEngine findValueByNumber(int number) {
+
+                return LayoutEngine.forNumber(number);
+            }
+        };
+
+
+        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
+            }
+            return getDescriptor().getValues()
+                    .get(ordinal());
+        }
+
+
+        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
+
+            return getDescriptor();
+        }
+
+
+        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
+
+            return LayoutEngineProto.getDescriptor().getEnumTypes()
+                    .get(0);
+        }
+
+
+        private static final LayoutEngine[] VALUES = values();
+
+
+        public static LayoutEngine valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+
+            if (desc.getType() != getDescriptor()) {
+                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
+            }
+            if (desc.getIndex() == -1) {
+                return UNRECOGNIZED;
+            }
+            return VALUES[desc.getIndex()];
+        }
+
+
+        private final int value;
+
+
+        private LayoutEngine(int value) {
+
+            this.value = value;
+        }
+
+        // @@protoc_insertion_point(enum_scope:LayoutEngine)
+    }
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\022LayoutEngine.proto*2\n\014LayoutEngine\022\r\n\t" + "ALGORITHM\020\000\022\006\n\002AI\020\001\022\013\n\007OUTLINE\020\002b\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
+        descriptor.resolveAllFeaturesImmutable();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 import java.io.Serializable;
 import java.util.Locale;
 
+@Deprecated
 public enum NodeType implements Serializable {
     DOCUMENT,
     SECTION,

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeTypeProto.java

@@ -0,0 +1,274 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.Locale;
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: NodeType.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class NodeTypeProto {
+
+    private NodeTypeProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", NodeTypeProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    /**
+     * Protobuf enum {@code NodeType}
+     */
+    public enum NodeType implements com.google.protobuf.ProtocolMessageEnum {
+        /**
+         * <code>DOCUMENT = 0;</code>
+         */
+        DOCUMENT(0),
+        /**
+         * <code>SECTION = 1;</code>
+         */
+        SECTION(1),
+        /**
+         * <code>SUPER_SECTION = 2;</code>
+         */
+        SUPER_SECTION(2),
+        /**
+         * <code>HEADLINE = 3;</code>
+         */
+        HEADLINE(3),
+        /**
+         * <code>PARAGRAPH = 4;</code>
+         */
+        PARAGRAPH(4),
+        /**
+         * <code>TABLE = 5;</code>
+         */
+        TABLE(5),
+        /**
+         * <code>TABLE_CELL = 6;</code>
+         */
+        TABLE_CELL(6),
+        /**
+         * <code>IMAGE = 7;</code>
+         */
+        IMAGE(7),
+        /**
+         * <code>HEADER = 8;</code>
+         */
+        HEADER(8),
+        /**
+         * <code>FOOTER = 9;</code>
+         */
+        FOOTER(9),
+        UNRECOGNIZED(-1),
+        ;
+
+
+        public String toString() {
+
+            return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+        }
+
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", NodeType.class.getName());
+        }
+
+        /**
+         * <code>DOCUMENT = 0;</code>
+         */
+        public static final int DOCUMENT_VALUE = 0;
+        /**
+         * <code>SECTION = 1;</code>
+         */
+        public static final int SECTION_VALUE = 1;
+        /**
+         * <code>SUPER_SECTION = 2;</code>
+         */
+        public static final int SUPER_SECTION_VALUE = 2;
+        /**
+         * <code>HEADLINE = 3;</code>
+         */
+        public static final int HEADLINE_VALUE = 3;
+        /**
+         * <code>PARAGRAPH = 4;</code>
+         */
+        public static final int PARAGRAPH_VALUE = 4;
+        /**
+         * <code>TABLE = 5;</code>
+         */
+        public static final int TABLE_VALUE = 5;
+        /**
+         * <code>TABLE_CELL = 6;</code>
+         */
+        public static final int TABLE_CELL_VALUE = 6;
+        /**
+         * <code>IMAGE = 7;</code>
+         */
+        public static final int IMAGE_VALUE = 7;
+        /**
+         * <code>HEADER = 8;</code>
+         */
+        public static final int HEADER_VALUE = 8;
+        /**
+         * <code>FOOTER = 9;</code>
+         */
+        public static final int FOOTER_VALUE = 9;
+
+
+        public final int getNumber() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
+            }
+            return value;
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         * @deprecated Use {@link #forNumber(int)} instead.
+         */
+        @Deprecated
+        public static NodeType valueOf(int value) {
+
+            return forNumber(value);
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         */
+        public static NodeType forNumber(int value) {
+
+            switch (value) {
+                case 0:
+                    return DOCUMENT;
+                case 1:
+                    return SECTION;
+                case 2:
+                    return SUPER_SECTION;
+                case 3:
+                    return HEADLINE;
+                case 4:
+                    return PARAGRAPH;
+                case 5:
+                    return TABLE;
+                case 6:
+                    return TABLE_CELL;
+                case 7:
+                    return IMAGE;
+                case 8:
+                    return HEADER;
+                case 9:
+                    return FOOTER;
+                default:
+                    return null;
+            }
+        }
+
+
+        public static com.google.protobuf.Internal.EnumLiteMap<NodeType> internalGetValueMap() {
+
+            return internalValueMap;
+        }
+
+
+        private static final com.google.protobuf.Internal.EnumLiteMap<NodeType> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<NodeType>() {
+            public NodeType findValueByNumber(int number) {
+
+                return NodeType.forNumber(number);
+            }
+        };
+
+
+        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
+            }
+            return getDescriptor().getValues()
+                    .get(ordinal());
+        }
+
+
+        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
+
+            return getDescriptor();
+        }
+
+
+        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
+
+            return NodeTypeProto.getDescriptor().getEnumTypes()
+                    .get(0);
+        }
+
+
+        private static final NodeType[] VALUES = values();
+
+
+        public static NodeType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+
+            if (desc.getType() != getDescriptor()) {
+                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
+            }
+            if (desc.getIndex() == -1) {
+                return UNRECOGNIZED;
+            }
+            return VALUES[desc.getIndex()];
+        }
+
+
+        private final int value;
+
+
+        private NodeType(int value) {
+
+            this.value = value;
+        }
+
+        // @@protoc_insertion_point(enum_scope:NodeType)
+    }
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\016NodeType.proto*\223\001\n\010NodeType\022\014\n\010DOCUMEN"
+                                   + "T\020\000\022\013\n\007SECTION\020\001\022\021\n\rSUPER_SECTION\020\002\022\014\n\010H"
+                                   + "EADLINE\020\003\022\r\n\tPARAGRAPH\020\004\022\t\n\005TABLE\020\005\022\016\n\nT"
+                                   + "ABLE_CELL\020\006\022\t\n\005IMAGE\020\007\022\n\n\006HEADER\020\010\022\n\n\006FO"
+                                   + "OTER\020\tb\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
+        descriptor.resolveAllFeaturesImmutable();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -19,6 +19,16 @@ public class SimplifiedText {
     @Schema(description = "Number of pages in the entire document.")
     private int numberOfPages;
     @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
+    @Builder.Default
     private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+    @Schema(description = "A list of the main section numbers ")
+    @Builder.Default
+    private List<String> mainSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the header section numbers ")
+    @Builder.Default
+    private List<String> headerSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the footer section numbers ")
+    @Builder.Default
+    private List<String> footerSectionNumbers = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,6 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
+    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
+    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
+    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPage.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+message AllDocumentPages {
+
+  repeated DocumentPage documentPages = 1;
+}
+
+message DocumentPage {
+  // The page number, starting with 1.
+  int32 number = 1;
+
+  // The page height in PDF user units.
+  int32 height = 2;
+
+  // The page width in PDF user units.
+  int32 width = 3;
+
+  // The page rotation as specified by the PDF.
+  int32 rotation = 4;
+}
+

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPositionData.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+message AllDocumentPositionData {
+
+  repeated DocumentPositionData documentPositionData = 1;
+}
+
+message DocumentPositionData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate.
+  // This is required due to the text and position coordinates not being equal.
+  repeated int32 stringIdxToPositionIdx = 2;
+
+  // The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block.
+  // The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner.
+  // In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.
+  repeated Position positions = 3;
+
+  // Definition of a BoundingBox that contains x, y, width, and height.
+  message Position {
+    repeated float value = 1;
+  }
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentStructure.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+import "EntryData.proto";
+
+message DocumentStructure {
+  // The root EntryData represents the Document.
+  EntryData root = 1;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentTextData.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+message AllDocumentTextData {
+
+  repeated DocumentTextData documentTextData = 1;
+}
+
+message DocumentTextData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // The page the text block occurs on.
+  int64 page = 2;
+
+  // The text of the text block.
+  string searchText = 3;
+
+  // Each text block is assigned a number on a page, starting from 0.
+  int32 numberOnPage = 4;
+
+  // The text blocks are ordered, this number represents the start of the text block as a string offset.
+  int32 start = 5;
+
+  // The text blocks are ordered, this number represents the end of the text block as a string offset.
+  int32 end = 6;
+
+  // The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.
+  repeated int32 lineBreaks = 7;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/EntryData.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+import "LayoutEngine.proto";
+import "NodeType.proto";
+
+message EntryData {
+  // Type of the semantic node.
+  NodeType type = 1;
+
+  // Specifies the position in the parsed tree structure.
+  repeated int32 treeId = 2;
+
+  // Specifies the text block IDs associated with this semantic node.
+  repeated int64 atomicBlockIds = 3;
+
+  // Specifies the pages this semantic node appears on.
+  repeated int64 pageNumbers = 4;
+
+  // Some semantic nodes have additional information, this information is stored in this Map.
+  map<string, string> properties = 5;
+
+  // All child Entries of this Entry.
+  repeated EntryData children = 6;
+
+  // Describes the origin of the semantic node.
+  repeated LayoutEngine engines = 7;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/LayoutEngine.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+enum LayoutEngine {
+  ALGORITHM = 0;
+  AI = 1;
+  OUTLINE = 2;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/NodeType.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+enum NodeType {
+  DOCUMENT = 0;
+  SECTION = 1;
+  SUPER_SECTION = 2;
+  HEADLINE = 3;
+  PARAGRAPH = 4;
+  TABLE = 5;
+  TABLE_CELL = 6;
+  IMAGE = 7;
+  HEADER = 8;
+  FOOTER = 9;
+}

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -12,12 +12,14 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
     implementation(project(":viewer-doc-processor"))
 
-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.564.0-RED9010.0") {
         exclude("org.springframework.boot", "spring-boot-starter-security")
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.30.0") {
+        exclude("com.iqser.red.commons", "storage-commons")
+    }
+    implementation("com.iqser.red.commons:storage-commons:2.50.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -25,7 +27,14 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
     implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
+    implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
+    implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
     implementation("org.tinspin:tinspin-indexes:2.1.3")
     implementation("org.commonmark:commonmark:0.22.0")
     implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")
+    implementation("com.pdftron:PDFNet:10.11.0")
+    implementation("org.apache.commons:commons-text:1.12.0")
+
+    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -2,13 +2,15 @@ package com.knecon.fforesight.service.layoutparser.processor;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Point2D;
+import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
@@ -20,11 +22,16 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.markdown.MarkdownMapper;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -39,7 +46,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableO
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
@@ -47,7 +54,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
@@ -57,9 +63,6 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
@@ -69,6 +72,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 import io.micrometer.observation.Observation;
 import io.micrometer.observation.ObservationRegistry;
@@ -90,10 +94,7 @@ public class LayoutParsingPipeline {
     CvTableParsingAdapter cvTableParsingAdapter;
     LayoutParsingStorageService layoutParsingStorageService;
     SectionsBuilderService sectionsBuilderService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
     SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
     RulingCleaningService rulingCleaningService;
     TableExtractionService tableExtractionService;
     DocuMineBlockificationService docuMineBlockificationService;
@@ -103,12 +104,12 @@ public class LayoutParsingPipeline {
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
     VisualLayoutParsingAdapter visualLayoutParsingAdapter;
-    ClarifyndClassificationService clarifyndClassificationService;
     GraphicExtractorService graphicExtractorService;
     OutlineExtractorService outlineExtractorService;
     OutlineValidationService outlineValidationService;
     TOCEnrichmentService tocEnrichmentService;
     LayoutparserSettings settings;
+    ClassificationService classificationService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -137,17 +138,16 @@ public class LayoutParsingPipeline {
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
         Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
-                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
-                                                           classificationDocument);
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent());
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
 
         log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
 
         layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        if(layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
+        if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
             layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(), new MarkdownMapper().toMarkdownContent(documentGraph));
         }
         layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
@@ -160,9 +160,9 @@ public class LayoutParsingPipeline {
         }
 
         if (!viewerDocumentFile.equals(originFile)) {
-            viewerDocumentFile.delete();
+            assert !viewerDocumentFile.exists() || viewerDocumentFile.delete();
         }
-        originFile.delete();
+        assert !originFile.exists() || originFile.delete();
 
         return LayoutParsingFinishedEvent.builder()
                 .identifier(layoutParsingRequest.identifier())
@@ -242,12 +242,8 @@ public class LayoutParsingPipeline {
         }
 
         List<ClassificationPage> classificationPages = new ArrayList<>();
-        OutlineObject lastProcessedOutlineObject = null;
 
-        // parsing the structure elements could be useful as well
-        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
-            classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
-        }
+        classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
 
         long pageCount = originDocument.getNumberOfPages();
 
@@ -271,21 +267,23 @@ public class LayoutParsingPipeline {
             stripper.setStartPage(pageNumber);
             stripper.setEndPage(pageNumber);
             stripper.setPdpage(pdPage);
-            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
-                stripper.setSortByPosition(true);
-            }
             stripper.getText(originDocument);
-            List<TextPositionSequence> words = stripper.getTextPositionSequences();
+            List<Word> words = stripper.getWords();
+
+//            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
+
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+                var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
+                classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
+                words = TextPositionOperations.sortWords(lines);
+            }
             classificationDocument.getLayoutDebugLayer().addTextVisualizations(words, pageNumber);
 
             PDRectangle pdr = pdPage.getMediaBox();
 
-            int rotation = pdPage.getRotation();
-            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-            PDRectangle cropbox = pdPage.getCropBox();
-            classificationDocument.getLayoutDebugLayer().addRulingVisualization(stripper.getRulings(), pageNumber);
-            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
+            List<Ruling> rulings = stripper.getRulings();
+            classificationDocument.getLayoutDebugLayer().addRulingVisualization(rulings, pageNumber);
+            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), rulings);
 
             PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
             List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), pageInformation);
@@ -293,7 +291,7 @@ public class LayoutParsingPipeline {
 
             TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);
 
-            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getTextPositionSequences(), false);
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);
 
             pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
                     .addAll(graphics.stream()
@@ -305,8 +303,7 @@ public class LayoutParsingPipeline {
                                     .toList());
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER_OLD ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                case REDACT_MANAGER_OLD -> redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                 case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                 case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                         docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -314,26 +311,9 @@ public class LayoutParsingPipeline {
                         docstrumBlockificationService.blockify(words, cleanRulings, false, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
             };
 
-            classificationPage.setCleanRulings(cleanRulings);
-            classificationPage.setRotation(rotation);
-            classificationPage.setLandscape(isLandscape);
-            classificationPage.setPageNumber(pageNumber);
-            classificationPage.setPageWidth(cropbox.getWidth());
-            classificationPage.setPageHeight(cropbox.getHeight());
+            updateClassificationPage(pdPage, pdr, classificationPage, cleanRulings, pageNumber, pageInformation);
 
-            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
-                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber - 1, new ArrayList<>());
-
-                OutlineObject notFoundOutlineObject = null;
-                if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
-                    lastProcessedOutlineObject.setPoint(new Point2D.Float(0, cropbox.getHeight()));
-                    notFoundOutlineObject = lastProcessedOutlineObject;
-                }
-                if (!outlineObjects.isEmpty()) {
-                    classificationPage.setOutlineObjects(outlineObjects);
-                    lastProcessedOutlineObject = blockificationPostprocessingService.sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
-                }
-            }
+            blockificationPostprocessingService.findHeadlinesFromOutline(classificationDocument, pageNumber, classificationPage, pageInformation);
 
             classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);
             // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
@@ -363,27 +343,9 @@ public class LayoutParsingPipeline {
 
         originDocument.close();
 
-        log.info("Calculating BodyTextFrame for {}", identifier);
-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        for (ClassificationPage page : classificationDocument.getPages()) {
-            classificationDocument.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
-        }
-        log.info("Classify TextBlocks for {}", identifier);
-        switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
-        }
+        classificationService.classify(classificationDocument, layoutParsingType, identifier);
 
-        List<TextPageBlock> headlines = classificationDocument.getPages()
-                .stream()
-                .flatMap(classificationPage -> classificationPage.getTextBlocks()
-                        .stream()
-                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
-                        .map(tb -> (TextPageBlock) tb))
-                .toList();
-        TableOfContents tableOfContents = outlineValidationService.createToC(headlines);
+        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
         classificationDocument.setTableOfContents(tableOfContents);
 
         log.info("Building Sections for {}", identifier);
@@ -397,6 +359,50 @@ public class LayoutParsingPipeline {
     }
 
 
+    private static void updateClassificationPage(PDPage pdPage,
+                                                 PDRectangle pdr,
+                                                 ClassificationPage classificationPage,
+                                                 CleanRulings cleanRulings,
+                                                 int pageNumber,
+                                                 PageInformation pageInformation) {
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+        classificationPage.setCleanRulings(cleanRulings);
+        classificationPage.setRotation(rotation);
+        classificationPage.setLandscape(isLandscape);
+        classificationPage.setPageNumber(pageNumber);
+        classificationPage.setPageWidth((float) pageInformation.width());
+        classificationPage.setPageHeight((float) pageInformation.height());
+    }
+
+
+    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
+
+        for (TextDirection dir : TextDirection.values()) {
+            double averageRotation = words.stream()
+                    .map(Word::getCharacters)
+                    .flatMap(Collection::stream)
+                    .map(Character::getTextPosition)
+                    .filter(pos -> pos.getDir().equals(dir))
+                    .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
+
+            if (averageRotation == 0) {
+                continue;
+            }
+
+            AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
+
+            for (Word word : words) {
+                if (!dir.equals(word.getDir())) {
+                    continue;
+                }
+                word.transform(rotateInstance);
+            }
+        }
+    }
+
+
     private void addNumberOfPagesToTrace(int numberOfPages, long size) {
 
         if (observationRegistry.getCurrentObservation() != null) {
@@ -438,10 +444,10 @@ public class LayoutParsingPipeline {
         // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
         for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
+                if (((TextPageBlock) textBlock).getWords() == null) {
                     continue;
                 }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                for (Word word : ((TextPageBlock) textBlock).getWords()) {
                     classificationPage.getTextHeightCounter().add(word.getTextHeight());
                     classificationPage.getFontCounter().add(word.getFont());
                     classificationPage.getFontSizeCounter().add(word.getFontSize());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -11,7 +11,9 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 
+import org.springframework.core.task.TaskExecutor;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -39,6 +41,8 @@ public class LayoutParsingStorageService {
     private final StorageService storageService;
     private final ObjectMapper objectMapper;
 
+    private final TaskExecutor taskExecutor;
+
 
     @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
     public File getOriginFile(String storageId) throws IOException {
@@ -100,13 +104,35 @@ public class LayoutParsingStorageService {
     }
 
 
+    @SneakyThrows
     @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
     public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
+        Runnable storeDocumentStructureRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                       layoutParsingRequest.structureFileStorageId(),
+                                                                                       documentData.getDocumentStructure());
+
+        CompletableFuture<Void> storeDocumentStructureFuture = CompletableFuture.runAsync(storeDocumentStructureRunnable, taskExecutor);
+
+        Runnable storeDocumentTextDataRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                      layoutParsingRequest.textBlockFileStorageId(),
+                                                                                      documentData.getDocumentTextData());
+
+        CompletableFuture<Void> storeDocumentTextDataFuture = CompletableFuture.runAsync(storeDocumentTextDataRunnable, taskExecutor);
+
+        Runnable storeDocumentPositionsRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                       layoutParsingRequest.positionBlockFileStorageId(),
+                                                                                       documentData.getDocumentPositions());
+
+        CompletableFuture<Void> storeDocumentPositionsFuture = CompletableFuture.runAsync(storeDocumentPositionsRunnable, taskExecutor);
+
+        Runnable storeDocumentPagesRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                   layoutParsingRequest.pageFileStorageId(),
+                                                                                   documentData.getDocumentPages());
+
+        CompletableFuture<Void> storeDocumentPagesFuture = CompletableFuture.runAsync(storeDocumentPagesRunnable, taskExecutor);
+
+        CompletableFuture.allOf(storeDocumentStructureFuture, storeDocumentTextDataFuture, storeDocumentPositionsFuture, storeDocumentPagesFuture).join();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum;
 
 import java.util.ArrayList;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -17,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.Zon
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 import lombok.RequiredArgsConstructor;
@@ -26,6 +27,7 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class DocstrumSegmentationService {
 
+    public static final double SAME_DIRECTION_THRESHOLD = 0.9;
     private final NearestNeighbourService nearestNeighbourService;
     private final SpacingService spacingService;
     private final LineBuilderService lineBuilderService;
@@ -33,30 +35,57 @@ public class DocstrumSegmentationService {
     private final ReadingOrderService readingOrderService;
 
 
-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
 
-        List<Zone> zones = new ArrayList<>();
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE));
+        EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);
 
-        return readingOrderService.resolve(zones, xyOrder);
+        List<Zone> newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO);
+        directionCounts.put(TextDirection.ZERO, newZones.size());
+        List<Zone> zones = new ArrayList<>(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE);
+        directionCounts.put(TextDirection.HALF_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.THREE_QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        return readingOrderService.resolve(zones, xyOrder, mostSameDirection(directionCounts));
     }
 
 
-    private List<Zone> computeZones(List<TextPositionSequence> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+    private boolean mostSameDirection(EnumMap<TextDirection, Integer> directionCounts) {
 
-        List<RedTextPosition> positions = textPositions.stream()
+        int total = directionCounts.values()
+                .stream()
+                .mapToInt(i -> i).sum();
+
+        if ((double) directionCounts.get(TextDirection.ZERO) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.HALF_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.THREE_QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        }
+        return false;
+    }
+
+
+    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+
+        List<Character> characters = textPositions.stream()
                 .filter(t -> t.getDir() == direction)
-                .map(TextPositionSequence::getTextPositions)
+                .map(Word::getCharacters)
                 .flatMap(List::stream)
                 .toList();
 
-        List<Character> characters = positions.stream()
-                .map(Character::new)
-                .collect(Collectors.toList());
-
         nearestNeighbourService.findNearestNeighbors(characters);
 
         double characterSpacing = spacingService.computeCharacterSpacing(characters);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/AngleFilter.java

@@ -15,10 +15,16 @@ public class AngleFilter {
 
     public boolean matches(Neighbor neighbor) {
 
+        return matches(neighbor.getAngle());
+    }
+
+
+    public boolean matches(double angle) {
+
         if (lowerAngle <= upperAngle) {
-            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle && angle < upperAngle;
         } else {
-            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle || angle < upperAngle;
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBox.java

@@ -7,8 +7,12 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
 
 @Data
+@SuperBuilder
+@NoArgsConstructor
 public abstract class BoundingBox {
 
     // Java coordinate system: (0, 0) is always upper left, x is increasing left to right and y is increasing from top to bottom.
@@ -19,7 +23,7 @@ public abstract class BoundingBox {
     // This rotates completely in 90 degree steps with page rotation.
     // Needs to be used when writing to a PDF.
     // Also, these are definitely correct and should be used whenever possible.
-    protected Rectangle2D bBoxInitialUserSpace;
+    protected Rectangle2D bBoxPdf;
 
     protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
 
@@ -50,25 +54,25 @@ public abstract class BoundingBox {
 
     public double getPdfMinX() {
 
-        return bBoxInitialUserSpace.getMinX();
+        return bBoxPdf.getMinX();
     }
 
 
     public double getPdfMaxX() {
 
-        return bBoxInitialUserSpace.getMaxX();
+        return bBoxPdf.getMaxX();
     }
 
 
     public double getPdfMinY() {
 
-        return bBoxInitialUserSpace.getMinY();
+        return bBoxPdf.getMinY();
     }
 
 
     public double getPdfMaxY() {
 
-        return bBoxInitialUserSpace.getMaxY();
+        return bBoxPdf.getMaxY();
     }
 
 
@@ -129,13 +133,31 @@ public abstract class BoundingBox {
     }
 
 
-    public boolean intersectsY(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other, float threshold) {
+
+        return this.getX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getX();
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other) {
+
+        return this.intersectsXPdf(other) && this.intersectsYPdf(other);
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXPdf(other, xThreshold) && this.intersectsYPdf(other, yThreshold);
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other) {
 
         return this.getPdfMinY() <= other.getPdfMaxY() && this.getPdfMaxY() >= other.getPdfMinY();
     }
 
 
-    public boolean intersectsYJava(BoundingBox other) {
+    public boolean intersectsY(BoundingBox other) {
 
         return this.getY() <= other.getMaxY() && this.getMaxY() >= other.getY();
     }
@@ -143,25 +165,31 @@ public abstract class BoundingBox {
 
     public boolean intersectsY(BoundingBox other, float threshold) {
 
+        return this.getY() - threshold <= other.getMaxY() && this.getMaxY() + threshold >= other.getY();
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other, float threshold) {
+
         return this.getPdfMinY() - threshold <= other.getPdfMaxY() && this.getPdfMaxY() + threshold >= other.getPdfMinY();
     }
 
 
-    public boolean intersectsX(BoundingBox other) {
+    public boolean intersectsXPdf(BoundingBox other) {
 
         return this.getPdfMinX() <= other.getPdfMaxX() && this.getPdfMaxX() >= other.getPdfMinX();
     }
 
 
-    public boolean intersectsXJava(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other) {
 
         return this.getX() <= other.getMaxX() && this.getMaxX() >= other.getMinX();
     }
 
 
-    public boolean intersectsX(BoundingBox other, float threshold) {
+    public boolean intersectsXPdf(BoundingBox other, float threshold) {
 
-        return this.getPdfMinX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
+        return this.getPdfMinX() - threshold <= other.getPdfMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
     }
 
 
@@ -170,8 +198,8 @@ public abstract class BoundingBox {
         this.bBox = components.stream()
                 .map(BoundingBox::getBBox)
                 .collect(RectangleTransformations.collectBBox());
-        this.bBoxInitialUserSpace = components.stream()
-                .map(BoundingBox::getBBoxInitialUserSpace)
+        this.bBoxPdf = components.stream()
+                .map(BoundingBox::getBBoxPdf)
                 .collect(RectangleTransformations.collectBBox());
     }
 
@@ -197,57 +225,55 @@ public abstract class BoundingBox {
 
     public double horizontalDistance(BoundingBox other) {
 
-        Rectangle2D left;
-        Rectangle2D right;
-        if (this.leftOf(other)) {
-            left = this.getBBox();
-            right = other.getBBox();
-        } else {
-            left = other.getBBox();
-            right = this.getBBox();
-        }
+        double rect1Right = getMaxX();
+        double rect1Left = getMinX();
+        double rect2Right = other.getMaxX();
+        double rect2Left = other.getMinX();
 
-        return Math.max(0, right.getMinX() - left.getMaxX());
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
     }
 
 
     public double verticalDistance(BoundingBox other) {
 
-        Rectangle2D bottom;
-        Rectangle2D top;
-        if (this.isAbove(other)) {
-            top = this.getBBox();
-            bottom = other.getBBox();
-        } else {
-            bottom = this.getBBox();
-            top = other.getBBox();
-        }
+        double rect1Top = getMaxY();
+        double rect1Bottom = getMinY();
+        double rect2Top = other.getMaxY();
+        double rect2Bottom = other.getMinY();
 
-        return Math.max(0, bottom.getMinY() - top.getMaxY());
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
     }
 
 
     public boolean rightOf(BoundingBox other) {
 
-        return this.intersectsYJava(other) && other.getMaxX() <= this.getMinX();
+        return this.intersectsY(other) && other.getMaxX() <= this.getMinX();
     }
 
 
     public boolean leftOf(BoundingBox other) {
 
-        return this.intersectsYJava(other) && other.getMinX() >= this.getMaxX();
+        return this.intersectsY(other) && other.getMinX() >= this.getMaxX();
     }
 
 
     public boolean isAbove(BoundingBox other) {
 
-        return this.intersectsXJava(other) && other.getMinY() >= this.getMaxY();
+        return this.intersectsX(other) && other.getMinY() >= this.getMaxY();
     }
 
 
     public boolean isBelow(BoundingBox other) {
 
-        return this.intersectsXJava(other) && this.getMinY() >= other.getMaxY();
+        return this.intersectsX(other) && this.getMinY() >= other.getMaxY();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Character.java

@@ -35,7 +35,7 @@ public class Character {
 
     public double getHeight() {
 
-        return textPosition.getHeightDir();
+        return textPosition.getHeightDirAdj();
     }
 
 
@@ -65,9 +65,9 @@ public class Character {
         double s = Math.sin(-0);
         double c = Math.cos(-0);
         xs[0] = c * x - s * y;
-        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDirAdj());
         xs[2] = c * other.x - s * other.y;
-        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDirAdj());
         boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
         Arrays.sort(xs);
         return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -1,18 +1,28 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;
+
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 @Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-public class Line extends BoundingBox {
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
+public class Line extends TextBoundingBox {
 
     private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
 
@@ -26,16 +36,13 @@ public class Line extends BoundingBox {
     @EqualsAndHashCode.Include
     private final double y1;
 
-    private final double height;
+    private FontStyle fontStyle;
 
-    private final List<Character> characters;
-    private final List<TextPositionSequence> words = new ArrayList<>();
+    private final List<Word> words;
 
 
     public Line(List<Character> characters, double wordSpacing) {
 
-        this.characters = characters;
-
         if (characters.size() >= 2) {
             // linear regression
             double sx = 0.0;
@@ -64,9 +71,43 @@ public class Line extends BoundingBox {
             this.y0 = character.getY() - dy;
             this.y1 = character.getY() + dy;
         }
-        height = computeHeight();
-        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
+        this.words = new ArrayList<>();
+        computeWords(characters, wordSpacing * WORD_DISTANCE_MULTIPLIER);
         buildBBox();
+        computeFontStyle();
+    }
+
+
+    public Line(List<Word> words) {
+
+        this.words = words;
+        buildBBox();
+        x0 = getMinX();
+        y0 = getMinY();
+        x1 = getMaxX();
+        y1 = getMaxY();
+        computeFontStyle();
+    }
+
+
+    private void computeFontStyle() {
+
+        EnumMap<FontStyle, AtomicInteger> fontStyleCounter = new EnumMap<>(FontStyle.class);
+        for (FontStyle fontStyle : FontStyle.values()) {
+            fontStyleCounter.put(fontStyle, new AtomicInteger(0));
+        }
+        for (Word word : words) {
+            switch (word.getFontStyle()) {
+                case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
+                case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
+                case ITALIC -> fontStyleCounter.get(FontStyle.ITALIC).getAndIncrement();
+                case BOLD_ITALIC -> fontStyleCounter.get(FontStyle.BOLD_ITALIC).getAndIncrement();
+            }
+        }
+        fontStyle = fontStyleCounter.entrySet()
+                .stream()
+                .max(Comparator.comparing(entry -> entry.getValue().get()))
+                .map(Map.Entry::getKey).orElse(FontStyle.REGULAR);
     }
 
 
@@ -82,14 +123,6 @@ public class Line extends BoundingBox {
     }
 
 
-    private double computeHeight() {
-
-        return characters.stream()
-                       .map(Character::getHeight)
-                       .reduce(0d, Double::sum) / characters.size();
-    }
-
-
     public double angularDifference(Line j) {
 
         double diff = Math.abs(getAngle() - j.getAngle());
@@ -122,19 +155,19 @@ public class Line extends BoundingBox {
     }
 
 
-    private void computeWords(double wordSpacing) {
+    private void computeWords(List<Character> characters, double wordSpacing) {
 
-        TextPositionSequence word = new TextPositionSequence();
+        Word word = new Word();
         Character previous = null;
         for (Character current : characters) {
             if (previous != null) {
                 double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
                 if (dist > wordSpacing) {
                     words.add(word);
-                    word = new TextPositionSequence();
+                    word = new Word();
                 }
             }
-            word.getTextPositions().add(current.getTextPosition());
+            word.add(current);
             previous = current;
         }
         words.add(word);
@@ -143,9 +176,7 @@ public class Line extends BoundingBox {
 
     private void buildBBox() {
 
-        this.setToBBoxOfComponents(characters.stream()
-                                       .map(Character::getTextPosition)
-                                       .toList());
+        this.setToBBoxOfComponents(words);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/TextBoundingBox.java

@@ -0,0 +1,180 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.SuperBuilder;
+
+@Getter
+@Setter
+@SuperBuilder
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+public abstract class TextBoundingBox extends BoundingBox {
+
+    protected Rectangle2D bBoxDirAdj;
+
+    protected TextDirection dir;
+
+
+    @Override
+    public void setToBBoxOfComponents(List<? extends BoundingBox> components) {
+
+        super.setToBBoxOfComponents(components);
+        this.bBoxDirAdj = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        Set<TextDirection> textDirections = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getDir)
+                .collect(Collectors.toSet());
+
+        if (textDirections.isEmpty()) {
+            dir = TextDirection.ZERO;
+        } else if (textDirections.size() > 1) {
+            throw new IllegalArgumentException("More than one text direction found");
+        } else {
+            dir = textDirections.iterator().next();
+        }
+    }
+
+
+    public double getXDirAdj() {
+
+        return this.bBoxDirAdj.getX();
+    }
+
+
+    public double getYDirAdj() {
+
+        return this.bBoxDirAdj.getY();
+    }
+
+
+    public double getWidthDirAdj() {
+
+        return this.bBoxDirAdj.getWidth();
+    }
+
+
+    public double getHeightDirAdj() {
+
+        return this.bBoxDirAdj.getHeight();
+    }
+
+
+    public double getMaxXDirAdj() {
+
+        return this.bBoxDirAdj.getMaxX();
+    }
+
+
+    public double getMaxYDirAdj() {
+
+        return this.bBoxDirAdj.getMaxY();
+    }
+
+
+    public double getCenterYDirAdj() {
+
+        return this.bBoxDirAdj.getCenterY();
+    }
+
+
+    public double getCenterXDirAdj() {
+
+        return this.bBoxDirAdj.getCenterX();
+    }
+
+
+    public double horizontalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Right = getMaxXDirAdj();
+        double rect1Left = getXDirAdj();
+        double rect2Right = other.getMaxXDirAdj();
+        double rect2Left = other.getXDirAdj();
+
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public double verticalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Top = getMaxYDirAdj();
+        double rect1Bottom = getYDirAdj();
+        double rect2Top = other.getMaxYDirAdj();
+        double rect2Bottom = other.getYDirAdj();
+
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.intersectsYDirAdj(other);
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXDirAdj(other, xThreshold) && this.intersectsYDirAdj(other, yThreshold);
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getXDirAdj() - threshold <= other.getMaxXDirAdj() && this.getMaxXDirAdj() + threshold >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other) {
+
+        return this.getXDirAdj() <= other.getMaxXDirAdj() && this.getMaxXDirAdj() >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other) {
+
+        return this.getYDirAdj() <= other.getMaxYDirAdj() && this.getMaxYDirAdj() >= other.getYDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
+    }
+
+
+    public boolean isAboveDirAdj(TextBoundingBox other) {
+
+        return other.isBelow(this);
+    }
+
+
+    public boolean isBelowDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/UnionFind.java

@@ -28,4 +28,10 @@ public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
         return setRep.values();
     }
 
+
+    public Collection<T> getElements() {
+
+        return getParentMap().keySet();
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Zone.java

@@ -6,9 +6,11 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 @Data
-public class Zone extends BoundingBox {
+@EqualsAndHashCode(callSuper = false)
+public class Zone extends TextBoundingBox {
 
     private List<Line> lines;
 
@@ -16,7 +18,6 @@ public class Zone extends BoundingBox {
     @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
     public Zone(List<Line> lines) {
 
-        lines.sort(Comparator.comparingDouble(Line::getY0));
         this.lines = lines;
         setToBBoxOfComponents(lines);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/LineBuilderService.java

@@ -17,7 +17,7 @@ public class LineBuilderService {
 
     private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
     private static final double LINE_SPACING_THRESHOLD_MULTIPLIER = 0.67;
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
 
 
     public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing, CleanRulings rulings) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ReadingOrderService.java

@@ -1,15 +1,17 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 
@@ -19,21 +21,30 @@ public class ReadingOrderService {
     private static final double THRESHOLD = 5;
     public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;
 
+    private static final Comparator<TextBoundingBox> COMPARATOR = //
+            Comparator.comparing(TextBoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
 
-    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder) {
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder, boolean useDirAdjCoords) {
 
         if (zones.isEmpty() || zones.size() == 1) {
             return zones;
         }
 
         if (xyReadingOrder) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
         }
 
         Map<Long, Integer> histogram = new HashMap<>();
         for (Zone zone : zones) {
-            long minY = Math.round(zone.getBBox().getMinY());
-            long maxY = Math.round(zone.getBBox().getMaxY());
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            long minY = Math.round(bbox.getMinY());
+            long maxY = Math.round(bbox.getMaxY());
             for (long i = minY; i <= maxY; i++) {
                 histogram.put(i, histogram.getOrDefault(i, 0) + 1);
             }
@@ -43,24 +54,32 @@ public class ReadingOrderService {
                     .stream()
                     .mapToInt(Integer::intValue).average()
                     .orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
         } else {
 
-            return resolveMultiColumnReadingOder(zones);
+            return resolveMultiColumnReadingOder(zones, useDirAdjCoords);
         }
 
     }
 
 
-    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones) {
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones, boolean useDirAdjCoords) {
 
-        zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                           .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            return zones.stream()
+                    .collect(Collectors.groupingBy(TextBoundingBox::getDir)).values()
+                    .stream()
+                    .flatMap(words -> words.stream()
+                            .sorted(COMPARATOR_DIR_ADJ))
+                    .toList();
+        }
+
+        zones.sort(COMPARATOR);
         return zones;
     }
 
 
-    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones, boolean useDirAdjCoords) {
 
         // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
         // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
@@ -69,11 +88,12 @@ public class ReadingOrderService {
         double maxX = Double.NEGATIVE_INFINITY;
 
         for (Zone zone : zones) {
-            if (zone.getX() < minX) {
-                minX = zone.getX();
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < minX) {
+                minX = zone.getXDirAdj();
             }
-            if (zone.getX() + zone.getWidth() > maxX) {
-                maxX = zone.getX() + zone.getWidth();
+            if (bbox.getMaxX() > maxX) {
+                maxX = zone.getMaxXDirAdj();
             }
         }
 
@@ -82,24 +102,27 @@ public class ReadingOrderService {
         List<Zone> leftOf = new ArrayList<>();
         List<Zone> rightOf = new ArrayList<>();
         List<Zone> middle = new ArrayList<>();
+
         for (Zone zone : zones) {
-            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < midLineXCoordinate && bbox.getX() + bbox.getWidth() < midLineXCoordinate) {
                 leftOf.add(zone);
-            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+            } else if (bbox.getX() > midLineXCoordinate && bbox.getX() + bbox.getWidth() > midLineXCoordinate) {
                 rightOf.add(zone);
             } else {
                 middle.add(zone);
             }
         }
 
-        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                             .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            leftOf.sort(COMPARATOR_DIR_ADJ);
+            rightOf.sort(COMPARATOR_DIR_ADJ);
+            middle.sort(COMPARATOR_DIR_ADJ);
+        } else {
+            leftOf.sort(COMPARATOR);
+            rightOf.sort(COMPARATOR);
+            middle.sort(COMPARATOR);
+        }
 /*
         List<Zone> leftNotIntersecting = new ArrayList<>();
         for (Zone leftZone : leftOf) {
@@ -151,8 +174,9 @@ public class ReadingOrderService {
 
         while (itty.hasNext()) {
             Zone current = itty.next();
+            Rectangle2D bbox = useDirAdjCoords ? current.getBBoxDirAdj() : current.getBBox();
             for (int i = 0; i < sortedZones.size(); i++) {
-                if (current.getY() < sortedZones.get(i).getY()) {
+                if (bbox.getY() < sortedZones.get(i).getY()) {
                     sortedZones.add(i, current);
                     itty.remove();
                     break;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -1,7 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
 import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -9,11 +9,12 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 @Service
 public class ZoneBuilderService {
@@ -21,7 +22,7 @@ public class ZoneBuilderService {
     private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
     private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
 
-    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -7;
 
     private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
 
@@ -29,7 +30,7 @@ public class ZoneBuilderService {
 
     private static final double MAX_LINE_SIZE_SCALE = 2.5;
 
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
 
     private static final double MAX_VERTICAL_MERGE_DISTANCE = 0.5;
 
@@ -38,7 +39,7 @@ public class ZoneBuilderService {
 
         double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
         double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
-        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = lineSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
         double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
 
         UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
@@ -54,11 +55,26 @@ public class ZoneBuilderService {
                     return;
                 }
 
-                double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
-                scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
+//                if (!innerLine.getFontStyle().equals(outerLine.getFontStyle()) //
+//                    && !outerLine.intersectsY(innerLine, -2f)) {
+//                    return;
+//                }
 
-                double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
-                double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
+                double horizontalScale = Math.min(outerLine.getHeightDirAdj(), innerLine.getHeightDirAdj()) / meanHeight;
+                horizontalScale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(horizontalScale, MAX_LINE_SIZE_SCALE));
+                double verticalScale = horizontalScale;
+
+//                if (innerLine.toString().endsWith(":")
+//                    || outerLine.toString().endsWith(":")
+//                    || numericalIdentifierPattern.matcher(innerLine.toString()).matches()
+//                    || numericalIdentifierPattern.matcher(outerLine.toString()).matches()) {
+//
+//                    horizontalScale *= 5;
+//                    verticalScale /= 10;
+//                }
+
+                double horizontalDistance = outerLine.horizontalDistance(innerLine) / horizontalScale;
+                double verticalDistance = outerLine.verticalDistance(innerLine) / verticalScale;
 
                 if ((!(minHorizontalDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalDistance)) //
                     && (!(minHorizontalMergeDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalMergeDistance))) {
@@ -87,7 +103,7 @@ public class ZoneBuilderService {
         double weights = 0.0;
         for (Line line : lines) {
             double weight = line.getLength();
-            meanHeight += line.getHeight() * weight;
+            meanHeight += line.getHeightDirAdj() * weight;
             weights += weight;
         }
         meanHeight /= weights;
@@ -97,64 +113,14 @@ public class ZoneBuilderService {
 
     private Zone mergeLinesInZone(List<Line> lines, double characterSpacing, double lineSpacing) {
 
-        double maxHorizontalDistance = 0;
-        double minVerticalDistance = 0;
-        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE;
+        Set<Word> words = lines.stream()
+                .map(Line::getWords)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toSet());
+        Collection<Set<Word>> groupedLines = TextPositionOperations.groupByLine(words);
 
-        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
-
-        lines.forEach(outer -> {
-            lines.forEach(inner -> {
-                if (inner == outer) {
-                    return;
-                }
-
-                double horizontalDistance = outer.horizontalDistance(inner);
-                double verticalDistance = outer.verticalDistance(inner);
-
-                if (horizontalDistance <= maxHorizontalDistance && minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance) {
-
-                    unionFind.union(outer, inner);
-
-                } else if (minVerticalDistance <= verticalDistance
-                           && verticalDistance <= maxVerticalDistance
-                           && Math.abs(horizontalDistance - Math.min(outer.getLength(), inner.getLength())) < 0.1) {
-
-                    boolean characterOverlap = false;
-                    int overlappingCount = 0;
-                    for (Character outerCharacter : outer.getCharacters()) {
-                        for (Character innerCharacter : inner.getCharacters()) {
-                            double characterOverlapDistance = outerCharacter.overlappingDistance(innerCharacter);
-                            if (characterOverlapDistance > 2) {
-                                characterOverlap = true;
-                            }
-                            if (characterOverlapDistance > 0) {
-                                overlappingCount++;
-                            }
-                        }
-                    }
-                    if (!characterOverlap && overlappingCount <= 2) {
-                        unionFind.union(outer, inner);
-                    }
-                }
-
-            });
-        });
-
-        List<Line> outputZone = new ArrayList<>();
-        for (Set<Line> group : unionFind.getGroups()) {
-            List<Character> characters = new ArrayList<>();
-            for (Line line : group) {
-                characters.addAll(line.getCharacters());
-            }
-            characters.sort(Comparator.comparingDouble(Character::getX));
-
-            outputZone.add(new Line(characters, characterSpacing));
-        }
-
-        return new Zone(outputZone.stream()
-                                .sorted(Comparator.comparing(Line::getY0))
-                                .collect(Collectors.toList()));
+        List<Line> sortedLines = TextPositionOperations.sortLines(groupedLines);
+        return new Zone(sortedLines);
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/DoubleUtils.java

@@ -7,9 +7,12 @@ public class DoubleUtils {
         if (Double.isNaN(d1) || Double.isNaN(d2)) {
             return Double.compare(d1, d2);
         }
-        long i1 = Math.round(d1 / (precision == 0 ? 1 : precision));
-        long i2 = Math.round(d2 / (precision == 0 ? 1 : precision));
-        return Long.compare(i1, i2);
+
+        if (Math.abs(d1 - d2) < precision) {
+            return 0;
+        }
+
+        return Double.compare(d1, d2);
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -4,7 +4,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -18,6 +18,7 @@ import lombok.RequiredArgsConstructor;
 
 @Data
 @RequiredArgsConstructor
+
 public class ClassificationPage {
 
     @NonNull
@@ -25,7 +26,7 @@ public class ClassificationPage {
 
     private List<OutlineObject> outlineObjects = new ArrayList<>();
 
-    private List<AbstractPageBlock> headlines  = new ArrayList<>();
+    private List<AbstractPageBlock> headlines = new ArrayList<>();
 
     private List<ClassifiedImage> images = new ArrayList<>();
 
@@ -44,7 +45,7 @@ public class ClassificationPage {
     private float pageWidth;
     private float pageHeight;
 
-    CleanRulings cleanRulings;
+    private CleanRulings cleanRulings;
 
     private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -12,10 +11,14 @@ import lombok.Getter;
 @Getter
 public class FloatFrequencyCounter {
 
-    Map<Float, Integer> countPerValue = new HashMap<>();
+    Map<Double, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    Double mostPopularCache;
 
 
-    public void add(float value) {
+    public void add(double value) {
+
+        changed = true;
 
         if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
@@ -25,9 +28,11 @@ public class FloatFrequencyCounter {
     }
 
 
-    public void addAll(Map<Float, Integer> otherCounter) {
+    public void addAll(Map<Double, Integer> otherCounter) {
 
-        for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
+        changed = true;
+
+        for (Map.Entry<Double, Integer> entry : otherCounter.entrySet()) {
             if (countPerValue.containsKey(entry.getKey())) {
                 countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
             } else {
@@ -37,36 +42,36 @@ public class FloatFrequencyCounter {
     }
 
 
-    public Float getMostPopular() {
+    public Double getMostPopular() {
 
-        Map.Entry<Float, Integer> mostPopular = null;
-        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed || mostPopularCache == null) {
+            Map.Entry<Double, Integer> mostPopular = null;
+            for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
             }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : 0;
+            changed = false;
         }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
     }
 
 
-    public List<Float> getHighterThanMostPopular() {
+    public List<Double> getValuesInReverseOrder() {
 
-        Float mostPopular = getMostPopular();
-        List<Float> higher = new ArrayList<>();
-        for (Float value : countPerValue.keySet()) {
-            if (value > mostPopular) {
-                higher.add(value);
-            }
-        }
-
-        return higher.stream().sorted(Collections.reverseOrder()).collect(Collectors.toList());
+        return countPerValue.keySet()
+                .stream()
+                .sorted(Collections.reverseOrder())
+                .collect(Collectors.toList());
     }
 
 
-    public Float getHighest() {
+    public Double getHighest() {
 
-        Float highest = null;
-        for (Float value : countPerValue.keySet()) {
+        Double highest = null;
+        for (Double value : countPerValue.keySet()) {
             if (highest == null || value > highest) {
                 highest = value;
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/LineInformation.java

@@ -3,7 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 public class LineInformation {
 
     List<Rectangle2D> lineBBox;
-    List<List<TextPositionSequence>> sequencesByLines;
+    List<List<Word>> sequencesByLines;
     List<List<Rectangle2D>> bBoxWithGapsByLines;
-    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+    List<List<List<Word>>> sequencesWithGapsByLines;
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -9,12 +9,14 @@ public enum PageBlockType {
     H6,
     HEADER,
     FOOTER,
-    TITLE,
     PARAGRAPH,
     PARAGRAPH_BOLD,
     PARAGRAPH_ITALIC,
     PARAGRAPH_UNKNOWN,
     OTHER,
+    TABLE_OF_CONTENTS_HEADLINE,
+    TABLE_OF_CONTENTS_ITEM,
+    LIST_ITEM,
     TABLE;
 
 
@@ -34,7 +36,7 @@ public enum PageBlockType {
     public static int getHeadlineNumber(PageBlockType pageBlockType) {
 
         return switch (pageBlockType) {
-            case H1 -> 1;
+            case H1, TABLE_OF_CONTENTS_HEADLINE -> 1;
             case H2 -> 2;
             case H3 -> 3;
             case H4 -> 4;
@@ -46,6 +48,6 @@ public enum PageBlockType {
 
     public boolean isHeadline() {
 
-        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6);
+        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6) || this.equals(TABLE_OF_CONTENTS_HEADLINE);
     }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageContents.java

@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -15,7 +15,7 @@ import lombok.Getter;
 @AllArgsConstructor
 public class PageContents {
 
-    List<TextPositionSequence> sortedTextPositionSequences;
+    List<Word> sortedWords;
     Rectangle2D cropBox;
     Rectangle2D mediaBox;
     List<Ruling> rulings;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifier.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -15,11 +16,13 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
 
-    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?");
 
     public enum Format {
         EMPTY,
         NUMERICAL,
+        ALPHANUMERIC,
         DOCUMENT
     }
 
@@ -41,6 +44,10 @@ public class SectionIdentifier {
         if (numericalIdentifierMatcher.find()) {
             return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
         }
+        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
+        if (alphanumericIdentifierMatcher.find()) {
+            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
+        }
         // more formats here
         return SectionIdentifier.empty();
     }
@@ -75,7 +82,36 @@ public class SectionIdentifier {
             }
             identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
         }
-        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+        return new SectionIdentifier(Format.NUMERICAL,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
+    }
+
+
+    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
+
+        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
+
+        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
+        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
+        List<Integer> identifiers = new LinkedList<>();
+        identifiers.add(mappedCharacterValue);
+
+        for (int i = 1; i <= 3; i++) {
+            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+
+        return new SectionIdentifier(Format.ALPHANUMERIC,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
     }
 
 
@@ -123,4 +159,22 @@ public class SectionIdentifier {
         return identifierString;
     }
 
+
+    public boolean isEmpty() {
+
+        return this.format.equals(Format.EMPTY);
+    }
+
+
+    public int level() {
+
+        return identifiers.size();
+    }
+
+
+    protected List<Integer> getIdentifiers() {
+
+        return identifiers;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/AbstractNodeVisitor.java

@@ -11,6 +11,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Se
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContentsItem;
 
 public abstract class AbstractNodeVisitor implements NodeVisitor {
 
@@ -83,8 +85,20 @@ public abstract class AbstractNodeVisitor implements NodeVisitor {
         visitChildren(tableCell);
     }
 
+    @Override
+    public void visit(TableOfContents toc) {
 
-    private void visitChildren(SemanticNode semanticNode) {
+        visitChildren(toc);
+    }
+
+    @Override
+    public void visit(TableOfContentsItem toci) {
+
+        visitChildren(toci);
+    }
+
+
+    protected void visitChildren(SemanticNode semanticNode) {
 
         semanticNode.streamChildren()
                 .forEach(node -> node.accept(this));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/DocumentTree.java

@@ -8,7 +8,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/NodeVisitor.java

@@ -10,6 +10,10 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Se
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContentsItem;
+
+import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
 
 public interface NodeVisitor {
 
@@ -42,4 +46,10 @@ public interface NodeVisitor {
 
     void visit(TableCell tableCell);
 
+
+    void visit(TableOfContents tableOfContents);
+
+
+    void visit(TableOfContentsItem tableOfContentsItem);
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/TextRange.java

@@ -98,10 +98,10 @@ public class TextRange implements Comparable<TextRange> {
     public List<TextRange> split(List<Integer> splitIndices) {
 
         if (splitIndices.stream()
-                .anyMatch(idx -> !this.containsExclusive(idx))) {
+                .anyMatch(idx -> !this.contains(idx))) {
             throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
                                                        splitIndices.stream()
-                                                               .filter(idx -> !this.containsExclusive(idx))
+                                                               .filter(idx -> !this.contains(idx))
                                                                .toList(),
                                                        this));
         }
@@ -116,7 +116,9 @@ public class TextRange implements Comparable<TextRange> {
             splitBoundaries.add(new TextRange(previousIndex, splitIndex));
             previousIndex = splitIndex;
         }
-        splitBoundaries.add(new TextRange(previousIndex, end));
+        if (previousIndex != end) {
+            splitBoundaries.add(new TextRange(previousIndex, end));
+        }
         return splitBoundaries;
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/AbstractSemanticNode.java

@@ -6,7 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -9,7 +9,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/DuplicatedParagraph.java

@@ -25,11 +25,4 @@ public class DuplicatedParagraph extends Paragraph {
 
     }
 
-
-    @Override
-    public String toString() {
-
-        return super.toString();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -6,7 +6,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -1,12 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
@@ -29,9 +32,8 @@ public class Page {
     Integer height;
     Integer width;
     Integer rotation;
-
     @EqualsAndHashCode.Exclude
-    List<SemanticNode> mainBody;
+    List<AtomicTextBlock> textBlocksOnPage;
     @EqualsAndHashCode.Exclude
     Header header;
     @EqualsAndHashCode.Exclude
@@ -53,20 +55,44 @@ public class Page {
                 .width((int) classificationPage.getPageWidth())
                 .number(classificationPage.getPageNumber())
                 .rotation(classificationPage.getRotation())
-                .mainBody(new LinkedList<>())
+                .textBlocksOnPage(new LinkedList<>())
                 .build();
     }
 
 
+    /**
+     * Constructs and returns a {@link TextBlock} representing the concatenated text of all leaf semantic nodes in the main body.
+     *
+     * @return The main body text block.
+     */
     public TextBlock getMainBodyTextBlock() {
 
-        return mainBody.stream()
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
+        return textBlocksOnPage.stream()
+                .filter(atb -> !atb.isEmpty())
                 .collect(new TextBlockCollector());
     }
 
 
+    public List<SemanticNode> getMainBody() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestParentOnPage)
+                .distinct()
+                .toList();
+    }
+
+
+    private SemanticNode getHighestParentOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
     @Override
     public String toString() {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java

@@ -12,8 +12,8 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
@@ -74,7 +74,8 @@ public interface SemanticNode {
 
         return getTextBlock().getPages()
                 .stream()
-                .min(Comparator.comparingInt(Page::getNumber)).orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
+                .min(Comparator.comparingInt(Page::getNumber))
+                .orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
     }
 
 
@@ -504,4 +505,17 @@ public interface SemanticNode {
 
     void accept(NodeVisitor visitor);
 
+
+    /**
+     * Checks wether this SemanticNode appears on a single page only, and if that page is the provided one.
+     *
+     * @param page the page to check
+     * @return true, when SemanticNode is on a single page only and the page is the provided page. Otherwise, false.
+     */
+    default boolean onlyOnPage(Page page) {
+
+        Set<Page> pages = getPages();
+        return pages.size() == 1 && pages.contains(page);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SuperSection.java

@@ -1,6 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java

@@ -12,8 +12,8 @@ import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java

@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableOfContents.java

@@ -0,0 +1,41 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class TableOfContents extends AbstractSemanticNode {
+
+    @Override
+    public NodeTypeProto.NodeType getType() {
+
+        return NodeTypeProto.NodeType.TABLE_OF_CONTENTS;
+    }
+
+
+    public Headline getHeadline() {
+
+        return streamChildrenOfType(NodeTypeProto.NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElseGet(() -> getParent().getHeadline());
+    }
+
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableOfContentsItem.java

@@ -0,0 +1,51 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class TableOfContentsItem extends AbstractSemanticNode {
+
+    TextBlock leafTextBlock;
+
+
+    @Override
+    public NodeTypeProto.NodeType getType() {
+
+        return NodeTypeProto.NodeType.TABLE_OF_CONTENTS_ITEM;
+    }
+
+
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        return leafTextBlock;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/AtomicTextBlock.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
@@ -13,8 +14,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
@@ -145,10 +146,7 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {
 
         return AtomicTextBlock.builder()
                 .id(documentTextData.getId())
@@ -156,9 +154,9 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(documentTextData.getStart(), documentTextData.getEnd()))
                 .searchText(documentTextData.getSearchText())
-                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed().toList())
-                .positions(toRectangle2DList(documentPositionData.getPositions()))
+                .lineBreaks(documentTextData.getLineBreaksList())
+                .stringIdxToPositionIdx(documentPositionData.getStringIdxToPositionIdxList())
+                .positions(toRectangle2DList(documentPositionData.getPositionsList()))
                 .parent(parent)
                 .build();
     }
@@ -166,7 +164,17 @@ public class AtomicTextBlock implements TextBlock {
 
     private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
 
-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
+        return Arrays.stream(positions)
+                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .toList();
+    }
+
+
+    private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {
+
+        return positions.stream()
+                .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getValue(0), pos.getValue(1), pos.getValue(2), pos.getValue(3)))
+                .toList();
     }
 
 
@@ -176,6 +184,9 @@ public class AtomicTextBlock implements TextBlock {
             throw new IndexOutOfBoundsException(format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
         }
         if (lineNumber == 0) {
+            if (lineBreaks.isEmpty()) {
+                return searchText;
+            }
             return subSequence(textRange.start(), lineBreaks.get(0) + textRange.start());
         } else if (lineNumber == numberOfLines() - 1) {
             return subSequence(lineBreaks.get(lineBreaks.size() - 1) + textRange.start(), textRange.end());
@@ -195,9 +206,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getNextLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + textRange.start();
+                       .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
+                       .findFirst() //
+                       .orElse(searchText.length()) + textRange.start();
     }
 
 
@@ -205,9 +216,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getPreviousLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + textRange.start();
+                       .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
+                       .reduce((a, b) -> b)//
+                       .orElse(0) + textRange.start();
     }
 
 
@@ -255,7 +266,10 @@ public class AtomicTextBlock implements TextBlock {
 
     protected List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
 
-        return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
+        return getLineBreaks().stream()
+                .map(linebreak -> linebreak + this.textRange.start())
+                .filter(textRange::contains)
+                .toList();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java

@@ -10,8 +10,8 @@ import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 
 @Data
-@RequiredArgsConstructor
 @AllArgsConstructor
+@RequiredArgsConstructor
 public class ClassifiedImage {
 
     @NonNull
@@ -20,11 +20,18 @@ public class ClassifiedImage {
     private ImageType imageType;
     private boolean sourceByAi;
     private boolean isAppendedToSection;
-    @NonNull
     private boolean hasTransparency;
-    @NonNull
     private int page;
-    @NonNull
     private String representation;
 
+
+    public ClassifiedImage(@NonNull Rectangle2D position, @NonNull ImageType imageType, boolean hasTransparency, int page, String representation) {
+
+        this.position = position;
+        this.imageType = imageType;
+        this.hasTransparency = hasTransparency;
+        this.page = page;
+        this.representation = representation;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineExtractorService.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
+import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -26,6 +27,9 @@ import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocume
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
@@ -82,11 +86,20 @@ public class OutlineExtractorService {
 
         String title = item.getTitle();
 
-        PDPage page = item.findDestinationPage(document);
-        if (page == null) {
+        PDPage page;
+        try {
+            // Can throw: "Error: can't convert to Destination COSArray" for some OCR'd PDFs
+            page = item.findDestinationPage(document);
+            if (page == null) {
+                return Optional.empty();
+            }
+        } catch (IOException e) {
+            log.info(String.format("Error occurred during position resolution for outline item with title %s: " + e, title));
             return Optional.empty();
         }
-        int pageNumber = document.getPages().indexOf(page);
+
+        int pageNumber = document.getPages().indexOf(page) + 1;
+        AffineTransform userSpaceToPageCoords = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(PageInformation.fromPDPage(pageNumber, page));
 
         Optional<Point2D> outlinePosition = Optional.empty();
 
@@ -115,8 +128,15 @@ public class OutlineExtractorService {
             log.info(String.format("Error occurred during position resolution for outline item on page %s with title %s: " + e, pageNumber, title));
         }
 
-        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title, pageNumber, outlinePosition.orElse(new Point2D.Float(0, 0)), depth)));
+        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title,
+                                                                       pageNumber,
+                                                                       transformPointToPageCoords(outlinePosition, userSpaceToPageCoords), depth)));
+    }
 
+
+    private static Point2D transformPointToPageCoords(Optional<Point2D> outlinePosition, AffineTransform userSpaceToPageCoords) {
+
+        return outlinePosition.map(point -> userSpaceToPageCoords.transform(point, null)).orElse(null);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObject.java

@@ -1,27 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.awt.geom.Point2D;
+import java.util.Optional;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+
+import lombok.Getter;
+import lombok.Setter;
 
-@Data
-@RequiredArgsConstructor
-@AllArgsConstructor
 public class OutlineObject {
 
+    @Getter
     private final String title;
+    @Getter
     private final int pageNumber;
-    private Point2D point;
+    @Getter
     private final int treeDepth;
 
+    private Point2D point; // java coordinates, (0, 0) is always top left
+
+    @Getter
+    @Setter
     private boolean found;
 
 
     public OutlineObject(String title, int pageNumber, Point2D point2D, int depth) {
 
-        this(title, pageNumber, depth);
+        this.title = title;
+        this.pageNumber = pageNumber;
+        this.treeDepth = depth;
         this.point = point2D;
     }
 
@@ -32,4 +39,39 @@ public class OutlineObject {
         return "OutlineObject{" + "title='" + title + '\'' + '}';
     }
 
+
+    public Optional<Point2D> getPoint() {
+
+        return Optional.ofNullable(point);
+    }
+
+
+    public boolean isAbove(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return true;
+        }
+        return point.getY() <= boundingBox.getMaxY();
+    }
+
+
+    public double distance(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return 0;
+        }
+        if (boundingBox.getBBox().contains(point)) {
+            return 0;
+        }
+        double deltaX = Math.min(Math.abs(boundingBox.getMinX() - point.getX()), Math.abs(boundingBox.getMaxX() - point.getX()));
+        double deltaY = Math.min(Math.abs(boundingBox.getMinY() - point.getY()), Math.abs(boundingBox.getMaxY() - point.getY()));
+        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+    }
+
+
+    public void resetPoint() {
+
+        this.point = null;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObjectTree.java

@@ -39,4 +39,28 @@ public class OutlineObjectTree {
         }
     }
 
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("OutlineObjectTree(\n");
+        for (OutlineObjectTreeNode node : rootNodes) {
+            buildString(node, sb, 1);
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    private void buildString(OutlineObjectTreeNode node, StringBuilder sb, int depth) {
+
+        for (int i = 0; i < depth; i++) {
+            sb.append("  ");
+        }
+        sb.append(node.getOutlineObject().getTitle()).append("\n");
+
+        for (OutlineObjectTreeNode child : node.getChildren()) {
+            buildString(child, sb, depth + 1);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineValidationService.java

@@ -10,6 +10,7 @@ import java.util.TreeSet;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import io.micrometer.observation.annotation.Observed;
@@ -20,7 +21,9 @@ import lombok.extern.slf4j.Slf4j;
 public class OutlineValidationService {
 
     @Observed(name = "OutlineValidationService", contextualName = "create-toc")
-    public TableOfContents createToC(List<TextPageBlock> headlines) {
+    public TableOfContents createToC(ClassificationDocument classificationDocument) {
+
+        List<TextPageBlock> headlines = extractHeadlines(classificationDocument);
 
         List<TableOfContentItem> mainSections = new ArrayList<>();
         Map<Integer, TableOfContentItem> lastItemsPerDepth = new HashMap<>();
@@ -60,4 +63,16 @@ public class OutlineValidationService {
         return new TableOfContents(mainSections);
     }
 
+
+    private static List<TextPageBlock> extractHeadlines(ClassificationDocument classificationDocument) {
+
+        return classificationDocument.getPages()
+                .stream()
+                .flatMap(classificationPage -> classificationPage.getTextBlocks()
+                        .stream()
+                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
+                        .map(tb -> (TextPageBlock) tb))
+                .toList();
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TOCEnrichmentService.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -167,7 +168,7 @@ public class TOCEnrichmentService {
             }
         }
 
-        if (!startBlocks.isEmpty()) {
+        if (!startBlocks.isEmpty() || !startImages.isEmpty()) {
             TableOfContentItem unassigned = new TableOfContentItem(null);
             unassigned.setSectionBlocks(startBlocks);
             unassigned.setImages(startImages);
@@ -185,12 +186,8 @@ public class TOCEnrichmentService {
             List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
             List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
             // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty()
-                && previousTable.getRowCount() == 1
-                && previousTable.getRows()
-                           .get(0).size() == tableNonHeaderRow.size()) {
-                previousTableNonHeaderRow = previousTable.getRows()
-                        .get(0)
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows().get(0)
                         .stream()
                         .map(cell -> {
                             Cell fakeCell = Cell.copy(cell);
@@ -201,8 +198,7 @@ public class TOCEnrichmentService {
             }
             if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                 for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                    List<Cell> row = currentTable.getRows()
-                            .get(i);
+                    List<Cell> row = currentTable.getRows().get(i);
                     if (row.size() == tableNonHeaderRow.size() && row.stream()
                             .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                         for (int j = 0; j < row.size(); j++) {
@@ -225,18 +221,15 @@ public class TOCEnrichmentService {
 
         return table.getRows()
                 .stream()
-                .flatMap(row -> row.stream()
-                        .filter(cell -> !cell.getHeaderCells().isEmpty()))
-                .findAny().isEmpty();
-
+                .flatMap(Collection::stream)
+                .allMatch(cell -> cell.getHeaderCells().isEmpty());
     }
 
 
     private List<Cell> getRowWithNonHeaderCells(TablePageBlock table) {
 
         for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-            List<Cell> row = table.getRows()
-                    .get(i);
+            List<Cell> row = table.getRows().get(i);
             if (row.size() == 1) {
                 continue;
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TableOfContentItem.java

@@ -5,7 +5,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
@@ -24,7 +24,7 @@ public class TableOfContentItem {
     private List<AbstractPageBlock> sectionBlocks = new ArrayList<>();
     private List<ClassifiedImage> images = new ArrayList<>();
 
-    private AbstractSemanticNode section;
+    private GenericSemanticNode section;
 
 
     public TableOfContentItem(TextPageBlock headline) {
@@ -45,8 +45,7 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index > 0) {
-                return parent.getChildren()
-                        .get(index - 1);
+                return parent.getChildren().get(index - 1);
             }
         }
         return null;
@@ -58,8 +57,7 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index >= 0 && index < parent.getChildren().size() - 1) {
-                return parent.getChildren()
-                        .get(index + 1);
+                return parent.getChildren().get(index + 1);
             }
         }
         return null;
@@ -93,17 +91,19 @@ public class TableOfContentItem {
         return false;
     }
 
+
     public List<AbstractPageBlock> getNonEmptySectionBlocks() {
 
-        return sectionBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
+        return sectionBlocks.stream()
+                .filter(pageBlock -> !pageBlock.isEmpty())
+                .collect(Collectors.toList());
     }
 
+
     @Override
     public String toString() {
 
         return "OutlineObjectTreeNode{" + "textPageBlock=" + headline + '}';
     }
 
-
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -9,7 +9,7 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Data;
@@ -35,14 +35,14 @@ public class Cell extends BoundingBox {
 
     public Cell(Point2D topLeft, Point2D bottomRight) {
 
-        this.bBoxInitialUserSpace = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
-        this.bBox = bBoxInitialUserSpace;
+        this.bBoxPdf = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
+        this.bBox = bBoxPdf;
     }
 
 
     public Cell(Rectangle2D bBoxInitialUserSpace, AffineTransform initialUserSpaceToJava) {
 
-        this.bBoxInitialUserSpace = bBoxInitialUserSpace;
+        this.bBoxPdf = bBoxInitialUserSpace;
         this.bBox = initialUserSpaceToJava.createTransformedShape(bBoxInitialUserSpace).getBounds2D();
     }
 
@@ -50,7 +50,7 @@ public class Cell extends BoundingBox {
     public static Cell copy(Cell cell) {
 
         Cell copy = new Cell();
-        copy.bBoxInitialUserSpace = cell.bBoxInitialUserSpace;
+        copy.bBoxPdf = cell.bBoxPdf;
         copy.bBox = cell.bBox;
         return copy;
     }
@@ -68,12 +68,12 @@ public class Cell extends BoundingBox {
         StringBuilder sb = new StringBuilder();
 
         Iterator<TextPageBlock> itty = textBlocks.iterator();
-        TextPositionSequence previous = null;
+        Word previous = null;
         while (itty.hasNext()) {
 
             TextPageBlock textBlock = itty.next();
 
-            for (TextPositionSequence word : textBlock.getSequences()) {
+            for (Word word : textBlock.getWords()) {
                 if (previous != null) {
                     if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');
@@ -87,7 +87,7 @@ public class Cell extends BoundingBox {
 
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
+        return TextNormalizationUtilities.cleanString(sb.toString());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/CleanRulings.java

@@ -70,7 +70,7 @@ public class CleanRulings {
 
     public boolean lineBetween(BoundingBox a, BoundingBox b) {
 
-        return lineBetween(a.getBBoxInitialUserSpace(), b.getBBoxInitialUserSpace());
+        return lineBetween(a.getBBoxPdf(), b.getBBoxPdf());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -263,8 +263,8 @@ public class TablePageBlock extends AbstractPageBlock {
 
                         cells.stream()
                                 .map(originalCell -> new CellWithIntersection(originalCell,
-                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxInitialUserSpace(),
-                                                                                                                                originalCell.getBBoxInitialUserSpace())))
+                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxPdf(),
+                                                                                                                                originalCell.getBBoxPdf())))
                                 .filter(cellWithIntersection -> cellWithIntersection.intersectedArea > 0)
                                 .filter(cellWithIntersection -> cellWithIntersection.originalCell.getArea() > cellWithIntersection.intersectedArea * CELL_AREA_CONTAINED_THRESHOLD)
                                 .max(Comparator.comparing(CellWithIntersection::intersectedArea))

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/AbstractBlockOnPage.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record AbstractBlockOnPage(AbstractPageBlock block, ClassificationPage page) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/FontStyle.java

@@ -0,0 +1,9 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+
+public enum FontStyle {
+    REGULAR,
+    BOLD,
+    ITALIC,
+    BOLD_ITALIC;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/FrequencyCounters.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@NoArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FrequencyCounters {
+
+    FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
+    StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
+    StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/ListIdentifier.java

@@ -0,0 +1,107 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ListIdentifier {
+
+    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]{1,4})\\.\\s+");
+    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]{1,4})\\)\\s+");
+
+    enum Format {
+        NUMBER_WITH_DOT,
+        NUMBER_IN_PARENTHESES
+    }
+
+    Format format;
+    @Getter
+    Word word;
+    @Getter
+    int page;
+    int representation;
+
+
+    public static Optional<ListIdentifier> parse(TextPageBlock textPageBlock, int page) {
+
+        return parse(textPageBlock.getWords().subList(0, Math.min(5, textPageBlock.getWords().size())), page);
+    }
+
+
+    public static Optional<ListIdentifier> parse(List<Word> sequences, int page) {
+
+        StringBuilder sb = new StringBuilder();
+        for (Word sequence : sequences) {
+            sb.append(sequence.toString());
+            sb.append(" ");
+        }
+        sb.replace(sb.length() - 1, sb.length(), "");
+        String text = sb.toString();
+
+        Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
+
+        if (numberMatcher.find()) {
+            Optional<Integer> representation = parseInteger(numberMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, representation.get()));
+            }
+        }
+
+        Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
+        if (parenthesisMatcher.find()) {
+            Optional<Integer> representation = parseInteger(parenthesisMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, representation.get()));
+            }
+        }
+        return Optional.empty();
+    }
+
+
+    private static Optional<Integer> parseInteger(String text) {
+
+        try {
+            return Optional.of(Integer.parseInt(text));
+        } catch (NumberFormatException e) {
+            return Optional.empty();
+        }
+    }
+
+
+    public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
+
+        if (listIdentifiers.size() <= 1) {
+            return true;
+        }
+
+        for (int i = 1; i < listIdentifiers.size(); i++) {
+            ListIdentifier current = listIdentifiers.get(i);
+            ListIdentifier previous = listIdentifiers.get(i - 1);
+            if (current.format != previous.format) {
+                return false;
+            }
+            if (current.representation <= previous.representation) {
+                return false;
+            }
+            if (!current.word.intersectsXDirAdj(previous.word, 2)) {
+                return false;
+            }
+            if (current.page == previous.page && !current.word.isBelowDirAdj(previous.word)) {
+                return false;
+            }
+            if (current.page < previous.page) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -5,64 +5,52 @@ import java.awt.geom.Rectangle2D;
 
 import org.apache.pdfbox.text.TextPosition;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 
+import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
 
 @Data
-@Builder
+@SuperBuilder
 @NoArgsConstructor
 @AllArgsConstructor
-public class RedTextPosition extends BoundingBox {
+@EqualsAndHashCode(callSuper = true)
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class RedTextPosition extends TextBoundingBox {
 
     public final static int HEIGHT_PADDING = 2;
 
-    private Rectangle2D.Float bBoxDirAdj; // adjusted to text rotation
+    String unicode;
 
-    @JsonIgnore
-    private int rotation;
+    // estimated using the TextMatrix in radians
+    float exactDir;
 
-    @JsonIgnore
-    private float pageHeight;
+    float widthOfSpace;
 
-    @JsonIgnore
-    private float pageWidth;
+    float fontSizeInPt;
 
-    private String unicode;
-
-    @JsonIgnore
-    private float dir;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float widthOfSpace;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float fontSizeInPt;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private String fontName;
+    String fontName;
 
 
     @SneakyThrows
     public static RedTextPosition fromTextPosition(TextPosition textPosition) {
 
         var pos = new RedTextPosition();
-        pos.setRotation(textPosition.getRotation());
-        pos.setPageHeight(textPosition.getPageHeight());
-        pos.setPageWidth(textPosition.getPageWidth());
         pos.setUnicode(textPosition.getUnicode());
-        pos.setDir(textPosition.getDir());
         pos.setWidthOfSpace(textPosition.getWidthOfSpace());
         pos.setFontSizeInPt(textPosition.getFontSizeInPt());
         pos.setFontName(textPosition.getFont().getName());
+        pos.setExactDir((float) FastAtan2.fastAtan2(textPosition.getTextMatrix().getShearY(), textPosition.getTextMatrix().getScaleX()));
+        pos.setDir(TextDirection.fromDegrees(textPosition.getDir()));
 
         //TODO: There is a mismatch in the java coords of the text and the rulings,
         // I guess if we start with the initial user space positions and transform them the same way we do the rulings it would work.
@@ -73,18 +61,18 @@ public class RedTextPosition extends BoundingBox {
                                                                  textPosition.getYDirAdj() - textHeight,
                                                                  textPosition.getWidthDirAdj(),
                                                                  textHeight + HEIGHT_PADDING);
+
         pos.setBBoxDirAdj(dirAdjPosition);
 
         AffineTransform affineTransform = getRotationMatrix(TextDirection.fromDegrees(textPosition.getDir()), textPosition.getPageWidth(), textPosition.getPageHeight());
         Rectangle2D bBoxInitialUserSpace = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
 
-        pos.setBBoxInitialUserSpace(bBoxInitialUserSpace); // These are definitely correct
+        pos.setBBoxPdf(bBoxInitialUserSpace); // These are definitely correct
 
         return pos;
     }
 
 
-
     private static AffineTransform getRotationMatrix(TextDirection textDirection, float pageWidth, float pageHeight) {
 
         AffineTransform transform = new AffineTransform();
@@ -103,32 +91,4 @@ public class RedTextPosition extends BoundingBox {
         return transform;
     }
 
-
-    @JsonIgnore
-    public float getXDirAdj() {
-
-        return this.bBoxDirAdj.x;
-    }
-
-
-    @JsonIgnore
-    public float getYDirAdj() {
-
-        return this.bBoxDirAdj.y;
-    }
-
-
-    @JsonIgnore
-    public float getWidthDirAdj() {
-
-        return this.bBoxDirAdj.width;
-    }
-
-
-    @JsonIgnore
-    public float getHeightDir() {
-
-        return this.bBoxDirAdj.height;
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Getter;
@@ -9,18 +10,18 @@ import lombok.Getter;
 @Getter
 public class SearchableText {
 
-    private final List<TextPositionSequence> sequences = new ArrayList<>();
+    private final List<Word> sequences = new ArrayList<>();
 
 
-    public void add(TextPositionSequence textPositionSequence) {
+    public void add(Word word) {
 
-        sequences.add(textPositionSequence);
+        sequences.add(word);
     }
 
 
-    public void addAll(List<TextPositionSequence> textPositionSequences) {
+    public void addAll(List<Word> words) {
 
-        sequences.addAll(textPositionSequences);
+        sequences.addAll(words);
     }
 
 
@@ -31,18 +32,14 @@ public class SearchableText {
     }
 
 
-    public static String buildString(List<TextPositionSequence> sequences) {
+    public static String buildString(List<Word> sequences) {
 
         StringBuilder sb = new StringBuilder();
-        for (TextPositionSequence word : sequences) {
+        for (Word word : sequences) {
             sb.append(word);
             sb.append(' ');
         }
-        String text = sb.toString();
-        text = TextNormalizationUtilities.removeHyphenLineBreaks(text);
-        text = TextNormalizationUtilities.removeLineBreaks(text);
-        text = TextNormalizationUtilities.removeRepeatingWhitespaces(text);
-        return text;
+        return TextNormalizationUtilities.cleanString(sb.toString());
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/StringFrequencyCounter.java

@@ -9,10 +9,14 @@ public class StringFrequencyCounter {
 
     @Getter
     private final Map<String, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    String mostPopularCache;
 
 
     public void add(String value) {
 
+        changed = true;
+
         if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
         } else {
@@ -23,6 +27,8 @@ public class StringFrequencyCounter {
 
     public void addAll(Map<String, Integer> otherCounter) {
 
+        changed = true;
+
         for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
             if (countPerValue.containsKey(entry.getKey())) {
                 countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
@@ -35,13 +41,18 @@ public class StringFrequencyCounter {
 
     public String getMostPopular() {
 
-        Map.Entry<String, Integer> mostPopular = null;
-        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed || mostPopularCache == null) {
+            Map.Entry<String, Integer> mostPopular = null;
+            for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
             }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
+            changed = false;
         }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextBlockOnPage.java

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record TextBlockOnPage(TextPageBlock textBlock, ClassificationPage page) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextDirection.java

@@ -44,4 +44,15 @@ public enum TextDirection {
 
         throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
     }
+
+
+    public int getRotation() {
+
+        return switch (this) {
+            case ZERO -> 0;
+            case QUARTER_CIRCLE -> 1;
+            case HALF_CIRCLE -> 2;
+            case THREE_QUARTER_CIRCLE -> 3;
+        };
+    }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -2,12 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.AllArgsConstructor;
@@ -24,62 +25,59 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {
 
     @Builder.Default
-    private List<TextPositionSequence> sequences = new ArrayList<>();
+    private List<Word> words = new ArrayList<>();
+    @Builder.Default
+    private FrequencyCounters frequencyCounters = new FrequencyCounters();
 
-    private String mostPopularWordFont;
+    private Rectangle2D bBoxDirAdj;
 
-    private String mostPopularWordStyle;
-
-    private float mostPopularWordFontSize;
-
-    private float mostPopularWordHeight;
-
-    private float mostPopularWordSpaceWidth;
-
-    private float highestFontSize;
+    private boolean underlined;
 
     private PageBlockType classification;
 
     private boolean toDuplicate;
 
+    private String text;
+    private boolean changed;
 
-    public TextPageBlock(List<TextPositionSequence> sequences) {
 
-        this.sequences = sequences;
-        calculateFrequencyCounters();
+    public TextPageBlock(List<Word> words) {
+
+        this.words = new ArrayList<>(words);
+        this.frequencyCounters = new FrequencyCounters();
+
+        if (!words.isEmpty()) {
+            addToFrequencyCounters(words);
+        }
         calculateBBox();
     }
 
 
-    @JsonIgnore
+    public List<Word> getWords() {
+
+        return Collections.unmodifiableList(words);
+    }
+
+
     public TextDirection getDir() {
 
-        return sequences.get(0).getDir();
-    }
-
-
-    @JsonIgnore
-    public float getPageHeight() {
-
-        return sequences.get(0).getPageHeight();
-    }
-
-
-    @JsonIgnore
-    public float getPageWidth() {
-
-        return sequences.get(0).getPageWidth();
+        return words.get(0).getDir();
     }
 
 
     private void calculateBBox() {
 
-        if (sequences == null) {
+        if (words == null) {
             this.bBox = new Rectangle2D.Double();
-            this.bBoxInitialUserSpace = new Rectangle2D.Double();
+            this.bBoxPdf = new Rectangle2D.Double();
+            this.bBoxDirAdj = new Rectangle2D.Double();
             return;
         }
-        setToBBoxOfComponents(sequences);
+        this.bBoxDirAdj = words.stream()
+                .map(Word::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        setToBBoxOfComponents(words);
     }
 
 
@@ -101,8 +99,8 @@ public class TextPageBlock extends AbstractPageBlock {
             throw new IllegalArgumentException("Cannot merge textBlocks on different pages.");
         }
 
-        List<TextPositionSequence> sequences = textBlocksToMerge.stream()
-                .map(TextPageBlock::getSequences)
+        List<Word> sequences = textBlocksToMerge.stream()
+                .map(TextPageBlock::getWords)
                 .flatMap(java.util.Collection::stream)
                 .toList();
         sequences = new ArrayList<>(sequences);
@@ -111,38 +109,27 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-    private void calculateFrequencyCounters() {
+    private void addToFrequencyCounters(List<Word> sequences) {
 
-        FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
-        StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
-        StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
-
-        for (TextPositionSequence wordBlock : sequences) {
-
-            lineHeightFrequencyCounter.add(wordBlock.getTextHeight());
-            fontSizeFrequencyCounter.add(wordBlock.getFontSize());
-            spaceFrequencyCounter.add(wordBlock.getSpaceWidth());
-            fontFrequencyCounter.add(wordBlock.getFont());
-            styleFrequencyCounter.add(wordBlock.getFontStyle());
+        for (Word wordBlock : sequences) {
 
+            frequencyCounters.getLineHeightFrequencyCounter().add(wordBlock.getTextHeight());
+            frequencyCounters.getFontSizeFrequencyCounter().add(wordBlock.getFontSize());
+            frequencyCounters.getSpaceFrequencyCounter().add(wordBlock.getSpaceWidth());
+            frequencyCounters.getFontFrequencyCounter().add(wordBlock.getFont());
+            frequencyCounters.getStyleFrequencyCounter().add(wordBlock.getFontStyle());
         }
 
-        setMostPopularWordFont(fontFrequencyCounter.getMostPopular());
-        setMostPopularWordStyle(styleFrequencyCounter.getMostPopular());
-        setMostPopularWordFontSize(fontSizeFrequencyCounter.getMostPopular());
-        setMostPopularWordHeight(lineHeightFrequencyCounter.getMostPopular());
-        setMostPopularWordSpaceWidth(spaceFrequencyCounter.getMostPopular());
-        setHighestFontSize(fontSizeFrequencyCounter.getHighest());
+        setUnderlined(this.words.stream()
+                              .allMatch(Word::isUnderline));
     }
 
 
-    public TextPageBlock union(TextPositionSequence r) {
+    public TextPageBlock union(Word r) {
 
         TextPageBlock union = this.copy();
-        union.getSequences().add(r);
-        calculateFrequencyCounters();
+        union.add(r);
+        addToFrequencyCounters(List.of(r));
         calculateBBox();
         return union;
     }
@@ -151,51 +138,50 @@ public class TextPageBlock extends AbstractPageBlock {
     public TextPageBlock union(TextPageBlock r) {
 
         TextPageBlock union = this.copy();
-        union.getSequences().addAll(r.getSequences());
-        calculateFrequencyCounters();
+        union.addAll(r.getWords());
+        addToFrequencyCounters(r.getWords());
         calculateBBox();
         return union;
     }
 
 
-    public void add(TextPageBlock r) {
+    public void add(TextPageBlock textPageBlock) {
 
-        sequences.addAll(r.getSequences());
-        calculateFrequencyCounters();
+        changed = true;
+        words.addAll(textPageBlock.getWords());
+        addToFrequencyCounters(textPageBlock.getWords());
         calculateBBox();
     }
 
 
-    public void add(TextPositionSequence r) {
+    public void add(Word word) {
 
-        sequences.add(r);
-        calculateFrequencyCounters();
+        changed = true;
+        words.add(word);
+        addToFrequencyCounters(List.of(word));
+        calculateBBox();
+    }
+
+
+    public void addAll(List<Word> words) {
+
+        changed = true;
+        this.words.addAll(words);
+        addToFrequencyCounters(words);
         calculateBBox();
     }
 
 
     public TextPageBlock copy() {
 
-        return new TextPageBlock(new ArrayList<>(sequences));
+        return new TextPageBlock(new ArrayList<>(words));
     }
 
 
     @Override
     public String toString() {
 
-        StringBuilder builder = new StringBuilder();
-
-        for (int i = 0; i < sequences.size(); i++) {
-            String sequenceAsString = sequences.get(i).toString();
-            // Fix for missing Whitespace. This is recognized in getSequences method. See PDFTextStripper Line 1730.
-            if (i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) != ' ' && sequenceAsString.charAt(0) != ' ') {
-                builder.append(' ');
-            }
-            builder.append(sequenceAsString);
-        }
-
-        return builder.toString();
-
+        return getText();
     }
 
 
@@ -203,30 +189,36 @@ public class TextPageBlock extends AbstractPageBlock {
     @JsonIgnore
     public String getText() {
 
-        StringBuilder sb = new StringBuilder();
+        if (text == null || changed) {
 
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
+            StringBuilder sb = new StringBuilder();
+
+            Word previous = null;
+            for (Word word : words) {
+                if (previous != null) {
+                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
+                        sb.append('\n');
+                    } else {
+                        sb.append(' ');
+                    }
                 }
+                sb.append(word.toString());
+                previous = word;
             }
-            sb.append(word.toString());
-            previous = word;
+
+            text = TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
+            changed = false;
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString());
+        return text;
     }
 
 
     public int getNumberOfLines() {
 
         int numberOfLines = 1;
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
+        Word previous = null;
+        for (Word word : words) {
             if (previous != null) {
                 if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
                     numberOfLines++;
@@ -238,10 +230,47 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
+    public String getMostPopularWordFont() {
+
+        return frequencyCounters.getFontFrequencyCounter().getMostPopular();
+    }
+
+
+    public String getMostPopularWordStyle() {
+
+        return frequencyCounters.getStyleFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordFontSize() {
+
+        return frequencyCounters.getFontSizeFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordHeight() {
+
+        return frequencyCounters.getLineHeightFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordSpaceWidth() {
+
+        return frequencyCounters.getSpaceFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getHighestFontSize() {
+
+        Double highest = frequencyCounters.getFontSizeFrequencyCounter().getHighest();
+        return highest == null ? 0 : highest;
+    }
+
+
     @Override
     public boolean isEmpty() {
 
-        return sequences.isEmpty();
+        return words.isEmpty();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -1,271 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.text.TextPosition;
-
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
-public class TextPositionSequence extends BoundingBox implements CharSequence {
-
-    public static final int HEIGHT_PADDING = 2;
-
-    @EqualsAndHashCode.Include
-    private int page;
-
-    @EqualsAndHashCode.Include
-    private List<RedTextPosition> textPositions = new ArrayList<>();
-
-    private Rectangle2D bBoxDirAdj;
-    @EqualsAndHashCode.Include
-    private TextDirection dir;
-    private int rotation;
-    private float pageHeight;
-    private float pageWidth;
-    private boolean isParagraphStart;
-    private boolean strikethrough;
-    private boolean underline;
-
-
-    public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
-
-        this.textPositions = textPositions.stream()
-                .map(RedTextPosition::fromTextPosition)
-                .collect(Collectors.toList());
-        this.page = pageNumber;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        this.isParagraphStart = isParagraphStart;
-        calculateBBox();
-    }
-
-
-    private void calculateBBox() {
-
-        this.bBoxDirAdj = textPositions.stream()
-                .map(RedTextPosition::getBBoxDirAdj)
-                .collect(RectangleTransformations.collectBBox());
-        setToBBoxOfComponents(getTextPositions());
-    }
-
-
-    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
-
-        this.textPositions = textPositions;
-        this.page = page;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
-    }
-
-
-    @Override
-    public int length() {
-
-        return textPositions.size();
-    }
-
-
-    @Override
-    public char charAt(int index) {
-
-        RedTextPosition textPosition = textPositionAt(index);
-        String text = textPosition.getUnicode();
-        return text.charAt(0);
-    }
-
-
-    public char charAt(int index, boolean caseInSensitive) {
-
-        RedTextPosition textPosition = textPositionAt(index);
-        String text = textPosition.getUnicode();
-        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
-    }
-
-
-    @Override
-    public TextPositionSequence subSequence(int start, int end) {
-
-        var textPositionSequence = new TextPositionSequence();
-        textPositionSequence.textPositions = textPositions.subList(start, end);
-        textPositionSequence.page = page;
-        textPositionSequence.dir = dir;
-        textPositionSequence.rotation = rotation;
-        textPositionSequence.pageHeight = pageHeight;
-        textPositionSequence.pageWidth = pageWidth;
-        textPositionSequence.setToBBoxOfComponents(getTextPositions());
-        return textPositionSequence;
-    }
-
-
-    @Override
-    public String toString() {
-
-        StringBuilder builder = new StringBuilder(length());
-        for (int i = 0; i < length(); i++) {
-            builder.append(charAt(i));
-        }
-        return builder.toString();
-    }
-
-
-    public RedTextPosition textPositionAt(int index) {
-
-        return textPositions.get(index);
-    }
-
-
-    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
-
-        this.textPositions.add(textPosition);
-        this.page = textPositionSequence.getPage();
-        this.dir = textPositionSequence.getDir();
-        this.rotation = textPositionSequence.getRotation();
-        this.pageHeight = textPositionSequence.getPageHeight();
-        this.pageWidth = textPositionSequence.getPageWidth();
-        calculateBBox();
-    }
-
-
-    public void add(TextPosition textPosition) {
-
-        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minX value
-     */
-
-    public float getMinXDirAdj() {
-
-        return textPositions.get(0).getXDirAdj();
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxX value
-     */
-
-    public float getMaxXDirAdj() {
-
-        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
-     */
-
-    public float getMinYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj() - getTextHeight();
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
-     */
-
-    public float getMaxYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj();
-
-    }
-
-
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
-    public float getTextHeight() {
-
-        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
-    }
-
-
-    public String getFont() {
-
-        if (textPositions.get(0).getFontName() == null) {
-            return "none";
-        }
-        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
-    }
-
-
-    public String getFontStyle() {
-
-        if (textPositions.get(0).getFontName() == null) {
-            return "standard";
-        }
-        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
-
-        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
-            return "bold, italic";
-        } else if (lowercaseFontName.contains("bold")) {
-            return "bold";
-        } else if (lowercaseFontName.contains("italic")) {
-            return "italic";
-        } else {
-            return "standard";
-        }
-    }
-
-
-    public float getFontSize() {
-
-        return textPositions.get(0).getFontSizeInPt();
-    }
-
-
-    public float getSpaceWidth() {
-
-        return textPositions.get(0).getWidthOfSpace();
-    }
-
-}
-

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequenceComparator.java

@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.Comparator;
+import java.util.HashMap;
+
+public class TextPositionSequenceComparator implements Comparator<Word> {
+
+    private HashMap<Word, TextBlockOnPage> lookup;
+
+
+    public TextPositionSequenceComparator(HashMap<Word, TextBlockOnPage> lookup) {
+
+        this.lookup = lookup;
+    }
+
+
+    @Override
+    public int compare(Word number1, Word number2) {
+
+        int page1 = lookup.get(number1).page().getPageNumber();
+        int page2 = lookup.get(number2).page().getPageNumber();
+
+        if (page1 != page2) {
+            return Integer.compare(page1, page2);
+        }
+
+        if (number1.getY() != number2.getY()) {
+            return Double.compare(number1.getY(), number2.getY());
+        }
+
+        return Integer.compare(Integer.parseInt(number1.toString()), Integer.parseInt(number2.toString()));
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/Word.java

@@ -0,0 +1,272 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
+
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.text.TextPosition;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@SuppressWarnings("pmd")
+public class Word extends TextBoundingBox implements CharSequence {
+
+    public static final String STANDARD = "standard";
+    public static final String BOLD_ITALIC = "bold, italic";
+    public static final String BOLD = "bold";
+    public static final String ITALIC = "italic";
+    public static final Pattern FONT_CLEANER = Pattern.compile(",bold|,italic");
+
+    private int page;
+
+    @Builder.Default
+    private List<Character> characters = new ArrayList<>();
+    private boolean isParagraphStart;
+    private boolean strikethrough;
+    private boolean underline;
+
+    private Integer hashcodeCache;
+
+
+    public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
+
+        this.characters = textPositions.stream()
+                .map(RedTextPosition::fromTextPosition)
+                .map(Character::new)
+                .collect(Collectors.toList());
+        this.page = pageNumber;
+        this.isParagraphStart = isParagraphStart;
+        calculateBBoxAndHashcode();
+    }
+
+
+    private void calculateBBoxAndHashcode() {
+
+        setToBBoxOfComponents(getTextPositions());
+        hashcodeCache = null;
+    }
+
+
+    public Word(List<Character> textPositions, int page) {
+
+        this.characters = new ArrayList<>(textPositions);
+        this.page = page;
+        calculateBBoxAndHashcode();
+    }
+
+
+    @Override
+    public int length() {
+
+        return characters.size();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        RedTextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return text.charAt(0);
+    }
+
+
+    public char charAt(int index, boolean caseInSensitive) {
+
+        RedTextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
+    }
+
+
+    @Override
+    public Word subSequence(int start, int end) {
+
+        var textPositionSequence = new Word();
+        textPositionSequence.characters = characters.subList(start, end);
+        textPositionSequence.page = page;
+        textPositionSequence.dir = dir;
+        textPositionSequence.setToBBoxOfComponents(getTextPositions());
+        return textPositionSequence;
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder builder = new StringBuilder(length());
+        for (int i = 0; i < length(); i++) {
+            builder.append(charAt(i));
+        }
+        return builder.toString();
+    }
+
+
+    public RedTextPosition textPositionAt(int index) {
+
+        return characters.get(index).getTextPosition();
+    }
+
+
+    public void add(Word word, RedTextPosition textPosition) {
+
+        this.characters.add(new Character(textPosition));
+        this.page = word.getPage();
+        calculateBBoxAndHashcode();
+    }
+
+
+    public void add(Character current) {
+
+        characters.add(current);
+        calculateBBoxAndHashcode();
+    }
+
+
+    public void add(TextPosition textPosition) {
+
+        add(new Character(RedTextPosition.fromTextPosition(textPosition)));
+    }
+
+
+    public double getTextHeightNoPadding() {
+
+        return characters.get(0).getTextPosition().getHeightDirAdj();
+    }
+
+
+    public double getTextHeight() {
+
+        return characters.get(0).getTextPosition().getHeightDirAdj() + HEIGHT_PADDING;
+    }
+
+
+    public String getFont() {
+
+        if (characters.get(0).getTextPosition().getFontName() == null) {
+            return "none";
+        }
+
+        return FONT_CLEANER.matcher(characters.get(0).getTextPosition().getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
+    }
+
+
+    public String getFontStyle() {
+
+        if (characters.get(0).getTextPosition().getFontName() == null) {
+            return STANDARD;
+        }
+        String lowercaseFontName = characters.get(0).getTextPosition().getFontName().toLowerCase(Locale.ROOT);
+
+        if (lowercaseFontName.contains(BOLD) && lowercaseFontName.contains(ITALIC)) {
+            return BOLD_ITALIC;
+        } else if (lowercaseFontName.contains(BOLD)) {
+            return BOLD;
+        } else if (lowercaseFontName.contains(ITALIC)) {
+            return ITALIC;
+        } else {
+            return STANDARD;
+        }
+    }
+
+
+    public float getFontSize() {
+
+        return characters.get(0).getTextPosition().getFontSizeInPt();
+    }
+
+
+    public float getSpaceWidth() {
+
+        return characters.get(0).getTextPosition().getWidthOfSpace();
+    }
+
+
+    public boolean equals(final Object o) {
+        // auto-generated with lombok
+        if (o == this) {
+            return true;
+        }
+        if (!(o instanceof Word other)) {
+            return false;
+        }
+        if (!other.canEqual((Object) this)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        if (this.getPage() != other.getPage()) {
+            return false;
+        }
+        final Object this$textPositions = this.getTextPositions();
+        final Object other$textPositions = other.getTextPositions();
+        if (!Objects.equals(this$textPositions, other$textPositions)) {
+            return false;
+        }
+        return Objects.equals(this.getHashcodeCache(), other.getHashcodeCache());
+    }
+
+
+    protected boolean canEqual(final Object other) {return other instanceof Word;}
+
+
+    public int hashCode() {
+
+        if (hashcodeCache == null) {
+            hashcodeCache = hashcodeCalculation();
+        }
+
+        return hashcodeCache;
+    }
+
+
+    private int hashcodeCalculation() {
+
+        final int PRIME = 59;
+        int result = super.hashCode();
+        result = result * PRIME + this.getPage();
+        final Object $textPositions = this.getTextPositions();
+        result = result * PRIME + ($textPositions == null ? 43 : $textPositions.hashCode());
+        return result;
+    }
+
+
+    private List<RedTextPosition> getTextPositions() {
+
+        return characters.stream()
+                .map(Character::getTextPosition)
+                .toList();
+    }
+
+
+    public void transform(AffineTransform rotateInstance) {
+
+        for (RedTextPosition textPosition : getTextPositions()) {
+            Rectangle2D exactDirAdjCoordinates = rotateInstance.createTransformedShape(textPosition.getBBoxDirAdj()).getBounds2D();
+            textPosition.setBBoxDirAdj(exactDirAdjCoordinates);
+        }
+        calculateBBoxAndHashcode();
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java

@@ -56,7 +56,7 @@ public class ImageServiceResponseAdapter {
         classificationPage.getImages().forEach(image -> {
             if (image.getImageType().equals(ImageType.OTHER)) {
                 for (AbstractPageBlock textblock : classificationPage.getTextBlocks()) {
-                    if (image.getPosition().contains(textblock.getBBoxInitialUserSpace())) {
+                    if (image.getPosition().contains(textblock.getBBoxPdf())) {
                         image.setImageType(ImageType.OCR);
                         return;
                     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class Classification {
 
+    @Builder.Default
     private Map<String, Float> probabilities = new HashMap<>();
     private String label;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -22,8 +22,10 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
+    @Builder.Default
     private List<ImageMetadata> data = new ArrayList<>();
 
+    @Builder.Default
     private List<ImageMetadata> dataCV = new ArrayList<>();
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -15,6 +15,7 @@ import lombok.NoArgsConstructor;
 public class TableData {
 
     private PageInfo pageInfo;
+    @Builder.Default
     private List<TableCells> tableCells = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -19,7 +19,7 @@ public class TableServiceResponse {
     private String operation;
     private String targetFileExtension;
     private String responseFileExtension;
-
+    @Builder.Default
     private List<TableData> data = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/DividingColumnDetectionService.java

@@ -23,11 +23,11 @@ public class DividingColumnDetectionService {
     public List<Rectangle2D> detectColumns(PageContents pageContents) {
 
 
-        if (pageContents.getSortedTextPositionSequences().size() < 2) {
+        if (pageContents.getSortedWords().size() < 2) {
             return List.of(pageContents.getCropBox());
         }
 
-        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), pageContents.getCropBox());
+        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), pageContents.getCropBox());
 
         return detectColumnsFromLines(linesWithGapInformation.getXGaps(), pageContents.getCropBox());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/GapDetectionService.java

@@ -5,8 +5,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 
 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -19,26 +18,26 @@ public class GapDetectionService {
     private static final double NEW_LINE_FACTOR = 0.2;
 
 
-    public static GapInformation findGapsInLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public static GapInformation findGapsInLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
 
-        if (sortedTextPositionSequences.isEmpty()) {
+        if (sortedWords.isEmpty()) {
             return new GapInformation();
         }
 
-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);
 
         XGapsContext xGapContext = XGapsContext.init(mainBodyTextFrame);
         YGapsContext yGapContext = YGapsContext.init(mainBodyTextFrame);
 
-        var previousTextPosition = sortedTextPositionSequences.get(0);
+        var previousTextPosition = sortedWords.get(0);
         Rectangle2D rectangle = toRectangle2D(previousTextPosition);
 
         xGapContext.addGapFromLeftEdgeOfMainBody(rectangle);
 
-        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
+        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
 
             double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
-            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getMinXDirAdj());
+            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
             Rectangle2D previousTextPositionBBox = toRectangle2D(previousTextPosition);
             Rectangle2D currentTextPositionBBox = toRectangle2D(currentTextPosition);
 
@@ -60,14 +59,14 @@ public class GapDetectionService {
             }
             previousTextPosition = currentTextPosition;
         }
-        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedTextPositionSequences.get(sortedTextPositionSequences.size() - 1)));
+        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedWords.get(sortedWords.size() - 1)));
         xGapContext.gapsPerLine.add(xGapContext.gapsInCurrentLine);
 
         return new GapInformation(xGapContext.gapsPerLine, yGapContext.gapsPerLine);
     }
 
 
-    private static Rectangle2D toRectangle2D(TextPositionSequence textPosition) {
+    private static Rectangle2D toRectangle2D(Word textPosition) {
 
         return mirrorY(textPosition.getBBox());
     }
@@ -88,18 +87,18 @@ public class GapDetectionService {
     }
 
 
-    private static void assertAllTextPositionsHaveSameDir(List<TextPositionSequence> textPositionSequences) {
+    private static void assertAllTextPositionsHaveSameDir(List<Word> words) {
 
-        assert textPositionSequences.stream()
-                .map(TextPositionSequence::getDir)
-                .allMatch(a -> a.equals(textPositionSequences.get(0).getDir()));
+        assert words.stream()
+                .map(Word::getDir)
+                .allMatch(a -> a.equals(words.get(0).getDir()));
     }
 
 
-    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
+    private static double getAvgTextPositionHeight(List<Word> words) {
 
-        return textPositionSequences.stream()
-                .mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
+        return words.stream()
+                .mapToDouble(Word::getHeight).average().orElseThrow();
     }