RED-9472: seperation of system rules

added some logs for investigation

.gitlab-ci.yml

@@ -7,20 +7,21 @@ include:
     ref: 'main'
     file: 'ci-templates/gradle_java.yml'
 
-deploy JavaDoc:
+publish dependencies:
   stage: deploy
   tags:
     - dind
   script:
-    - echo "Building JavaDoc with gradle version ${BUILDVERSION}"
+    - echo "Publishing dependencies with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
     - echo "BUILDVERSION=$(echo ${BUILDVERSION})" >> variables.env
   artifacts:
     reports:
-        dotenv: variables.env
+      dotenv: variables.env
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_BRANCH =~ /^feature/
     - if: $CI_COMMIT_TAG
 
 generate JavaDoc:
@@ -42,7 +43,7 @@ pages:
   stage: deploy
   needs:
     - generate JavaDoc
-    - deploy JavaDoc
+    - publish dependencies
     - calculate minor version
   pages:
     path_prefix: "$BUILDVERSION"

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -11,12 +11,19 @@ group = "com.iqser.red.service"
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
+val persistenceServiceVersion by rootProject.extra { "2.651.0-RED9472.1" }
+
 pmd {
     isConsoleOutput = true
 }
 
+tasks.checkstyleMain {
+    exclude("**/data/**") // ignore generated proto files
+}
+
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+    exclude("**/data/**") // ignore generated proto files
 }
 
 tasks.pmdTest {
@@ -60,3 +67,13 @@ repositories {
         }
     }
 }
+
+allprojects {
+    tasks.withType<Javadoc> {
+        options {
+            this as StandardJavadocDocletOptions
+            addBooleanOption("Xdoclint:none", true)
+            addStringOption("Xmaxwarns", "1")
+        }
+    }
+}

publish-custom-image.sh

@@ -9,11 +9,14 @@ gradle assemble
 # Get the current Git branch
 branch=$(git rev-parse --abbrev-ref HEAD)
 
+# Replace any slashes (e.g., in 'feature/' or 'release/') with a hyphen
+cleaned_branch=$(echo "$branch" | sed 's/\//_/g')
+
 # Get the short commit hash (first 5 characters)
 commit_hash=$(git rev-parse --short=5 HEAD)
 
 # Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
+buildName="${USER}-${cleaned_branch}-${commit_hash}"
 
 gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}
 

redaction-service-v1/document/build.gradle.kts

@@ -0,0 +1,36 @@
+plugins {
+    id("com.iqser.red.service.java-conventions")
+    id("io.freefair.lombok") version "8.4"
+}
+
+description = "redaction-service-document"
+
+
+group = "com.knecon.fforesight"
+
+dependencies {
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${rootProject.extra.get("persistenceServiceVersion")}") {
+        exclude(group = "com.knecon.fforesight", module = "document")
+    }
+    api("com.google.protobuf:protobuf-java-util:4.28.3")
+
+    testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1")
+    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")
+}
+
+publishing {
+    publications {
+        create<MavenPublication>(name) {
+            from(components["java"])
+        }
+    }
+    repositories {
+        maven {
+            url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+            credentials {
+                username = providers.gradleProperty("mavenUser").getOrNull();
+                password = providers.gradleProperty("mavenPassword").getOrNull();
+            }
+        }
+    }
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/DocumentData.java

@@ -0,0 +1,36 @@
+package com.iqser.red.service.redaction.v1.server.data;
+
+import static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure;
+
+import java.io.Serializable;
+
+import com.iqser.red.service.redaction.v1.server.data.DocumentPageProto.AllDocumentPages;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.AllDocumentPositionData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto.AllDocumentTextData;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentData implements Serializable {
+
+    AllDocumentPages documentPages;
+    AllDocumentTextData documentTextData;
+    AllDocumentPositionData documentPositionData;
+    DocumentStructureWrapper documentStructureWrapper;
+
+
+    public DocumentStructure getDocumentStructure() {
+
+        return documentStructureWrapper.getDocumentStructure();
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/DocumentStructureProto.java

@@ -0,0 +1,694 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: DocumentStructure.proto
+// Protobuf Java Version: 4.28.3
+
+package com.iqser.red.service.redaction.v1.server.data;
+
+public final class DocumentStructureProto {
+  private DocumentStructureProto() {}
+  static {
+    com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+      com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+      /* major= */ 4,
+      /* minor= */ 28,
+      /* patch= */ 3,
+      /* suffix= */ "",
+      DocumentStructureProto.class.getName());
+  }
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistryLite registry) {
+  }
+
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+    registerAllExtensions(
+        (com.google.protobuf.ExtensionRegistryLite) registry);
+  }
+  public interface DocumentStructureOrBuilder extends
+      // @@protoc_insertion_point(interface_extends:DocumentStructure)
+      com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     * @return Whether the root field is set.
+     */
+    boolean hasRoot();
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     * @return The root.
+     */
+    com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData getRoot();
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     */
+    com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder getRootOrBuilder();
+  }
+  /**
+   * Protobuf type {@code DocumentStructure}
+   */
+  public static final class DocumentStructure extends
+      com.google.protobuf.GeneratedMessage implements
+      // @@protoc_insertion_point(message_implements:DocumentStructure)
+      DocumentStructureOrBuilder {
+  private static final long serialVersionUID = 0L;
+    static {
+      com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+        com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+        /* major= */ 4,
+        /* minor= */ 28,
+        /* patch= */ 3,
+        /* suffix= */ "",
+        DocumentStructure.class.getName());
+    }
+    // Use DocumentStructure.newBuilder() to construct.
+    private DocumentStructure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+      super(builder);
+    }
+    private DocumentStructure() {
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.class, com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.Builder.class);
+    }
+
+    private int bitField0_;
+    public static final int ROOT_FIELD_NUMBER = 1;
+    private com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData root_;
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     * @return Whether the root field is set.
+     */
+    @java.lang.Override
+    public boolean hasRoot() {
+      return ((bitField0_ & 0x00000001) != 0);
+    }
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     * @return The root.
+     */
+    @java.lang.Override
+    public com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData getRoot() {
+      return root_ == null ? com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.getDefaultInstance() : root_;
+    }
+    /**
+     * <pre>
+     * The root EntryData represents the Document.
+     * </pre>
+     *
+     * <code>.EntryData root = 1;</code>
+     */
+    @java.lang.Override
+    public com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+      return root_ == null ? com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.getDefaultInstance() : root_;
+    }
+
+    private byte memoizedIsInitialized = -1;
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      if (((bitField0_ & 0x00000001) != 0)) {
+        output.writeMessage(1, getRoot());
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (((bitField0_ & 0x00000001) != 0)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(1, getRoot());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure)) {
+        return super.equals(obj);
+      }
+      com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure other = (com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure) obj;
+
+      if (hasRoot() != other.hasRoot()) return false;
+      if (hasRoot()) {
+        if (!getRoot()
+            .equals(other.getRoot())) return false;
+      }
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      if (hasRoot()) {
+        hash = (37 * hash) + ROOT_FIELD_NUMBER;
+        hash = (53 * hash) + getRoot().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        java.nio.ByteBuffer data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        java.nio.ByteBuffer data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+    public static Builder newBuilder(com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE
+          ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /**
+     * Protobuf type {@code DocumentStructure}
+     */
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder> implements
+        // @@protoc_insertion_point(builder_implements:DocumentStructure)
+        com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructureOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.class, com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.Builder.class);
+      }
+
+      // Construct using com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(
+          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage
+                .alwaysUseFieldBuilders) {
+          getRootFieldBuilder();
+        }
+      }
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        root_ = null;
+        if (rootBuilder_ != null) {
+          rootBuilder_.dispose();
+          rootBuilder_ = null;
+        }
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure getDefaultInstanceForType() {
+        return com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure build() {
+        com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure buildPartial() {
+        com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure result = new com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure(this);
+        if (bitField0_ != 0) { buildPartial0(result); }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartial0(com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure result) {
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) != 0)) {
+          result.root_ = rootBuilder_ == null
+              ? root_
+              : rootBuilder_.build();
+          to_bitField0_ |= 0x00000001;
+        }
+        result.bitField0_ |= to_bitField0_;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure) {
+          return mergeFrom((com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure other) {
+        if (other == com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure.getDefaultInstance()) return this;
+        if (other.hasRoot()) {
+          mergeRoot(other.getRoot());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 10: {
+                input.readMessage(
+                    getRootFieldBuilder().getBuilder(),
+                    extensionRegistry);
+                bitField0_ |= 0x00000001;
+                break;
+              } // case 10
+              default: {
+                if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                  done = true; // was an endgroup tag
+                }
+                break;
+              } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+      private int bitField0_;
+
+      private com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData root_;
+      private com.google.protobuf.SingleFieldBuilder<
+          com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.Builder, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder> rootBuilder_;
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       * @return Whether the root field is set.
+       */
+      public boolean hasRoot() {
+        return ((bitField0_ & 0x00000001) != 0);
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       * @return The root.
+       */
+      public com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData getRoot() {
+        if (rootBuilder_ == null) {
+          return root_ == null ? com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.getDefaultInstance() : root_;
+        } else {
+          return rootBuilder_.getMessage();
+        }
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public Builder setRoot(com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData value) {
+        if (rootBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          root_ = value;
+        } else {
+          rootBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return this;
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public Builder setRoot(
+          com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.Builder builderForValue) {
+        if (rootBuilder_ == null) {
+          root_ = builderForValue.build();
+        } else {
+          rootBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return this;
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public Builder mergeRoot(com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData value) {
+        if (rootBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) != 0) &&
+            root_ != null &&
+            root_ != com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.getDefaultInstance()) {
+            getRootBuilder().mergeFrom(value);
+          } else {
+            root_ = value;
+          }
+        } else {
+          rootBuilder_.mergeFrom(value);
+        }
+        if (root_ != null) {
+          bitField0_ |= 0x00000001;
+          onChanged();
+        }
+        return this;
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public Builder clearRoot() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        root_ = null;
+        if (rootBuilder_ != null) {
+          rootBuilder_.dispose();
+          rootBuilder_ = null;
+        }
+        onChanged();
+        return this;
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.Builder getRootBuilder() {
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return getRootFieldBuilder().getBuilder();
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      public com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+        if (rootBuilder_ != null) {
+          return rootBuilder_.getMessageOrBuilder();
+        } else {
+          return root_ == null ?
+              com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.getDefaultInstance() : root_;
+        }
+      }
+      /**
+       * <pre>
+       * The root EntryData represents the Document.
+       * </pre>
+       *
+       * <code>.EntryData root = 1;</code>
+       */
+      private com.google.protobuf.SingleFieldBuilder<
+          com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.Builder, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder> 
+          getRootFieldBuilder() {
+        if (rootBuilder_ == null) {
+          rootBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData.Builder, com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryDataOrBuilder>(
+                  getRoot(),
+                  getParentForChildren(),
+                  isClean());
+          root_ = null;
+        }
+        return rootBuilder_;
+      }
+
+      // @@protoc_insertion_point(builder_scope:DocumentStructure)
+    }
+
+    // @@protoc_insertion_point(class_scope:DocumentStructure)
+    private static final com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure DEFAULT_INSTANCE;
+    static {
+      DEFAULT_INSTANCE = new com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure();
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser<DocumentStructure>
+        PARSER = new com.google.protobuf.AbstractParser<DocumentStructure>() {
+      @java.lang.Override
+      public DocumentStructure parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        Builder builder = newBuilder();
+        try {
+          builder.mergeFrom(input, extensionRegistry);
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.setUnfinishedMessage(builder.buildPartial());
+        } catch (com.google.protobuf.UninitializedMessageException e) {
+          throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial());
+        } catch (java.io.IOException e) {
+          throw new com.google.protobuf.InvalidProtocolBufferException(e)
+              .setUnfinishedMessage(builder.buildPartial());
+        }
+        return builder.buildPartial();
+      }
+    };
+
+    public static com.google.protobuf.Parser<DocumentStructure> parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<DocumentStructure> getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+
+  }
+
+  private static final com.google.protobuf.Descriptors.Descriptor
+    internal_static_DocumentStructure_descriptor;
+  private static final 
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_DocumentStructure_fieldAccessorTable;
+
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static  com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\027DocumentStructure.proto\032\017EntryData.pro" +
+      "to\"-\n\021DocumentStructure\022\030\n\004root\030\001 \001(\0132\n." +
+      "EntryDataBH\n.com.iqser.red.service.redac" +
+      "tion.v1.server.dataB\026DocumentStructurePr" +
+      "otob\006proto3"
+    };
+    descriptor = com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+          com.iqser.red.service.redaction.v1.server.data.EntryDataProto.getDescriptor(),
+        });
+    internal_static_DocumentStructure_descriptor =
+      getDescriptor().getMessageTypes().get(0);
+    internal_static_DocumentStructure_fieldAccessorTable = new
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+        internal_static_DocumentStructure_descriptor,
+        new java.lang.String[] { "Root", });
+    descriptor.resolveAllFeaturesImmutable();
+    com.iqser.red.service.redaction.v1.server.data.EntryDataProto.getDescriptor();
+  }
+
+  // @@protoc_insertion_point(outer_class_scope)
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/DocumentStructureWrapper.java

@@ -0,0 +1,115 @@
+package com.iqser.red.service.redaction.v1.server.data;
+
+import static com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure;
+import static com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData;
+
+import java.awt.geom.Rectangle2D;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@Getter
+@AllArgsConstructor
+public class DocumentStructureWrapper implements Serializable {
+
+    private final DocumentStructure documentStructure;
+
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+        public static final String REPRESENTATION_HASH = "representationHash";
+
+    }
+
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public static double[] parseRepresentationVector(String representationHash) {
+
+        String[] stringArray = representationHash.split("[,\\s]+");
+        double[] doubleArray = new double[stringArray.length];
+        for (int i = 0; i < stringArray.length; i++) {
+            doubleArray[i] = Double.parseDouble(stringArray[i]);
+        }
+
+        return doubleArray;
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return documentStructure.getRoot();
+        }
+        EntryData entry = documentStructure.getRoot().getChildrenList()
+                .get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.getChildrenList()
+                    .get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return flatten(documentStructure.getRoot());
+    }
+
+
+    public String toString() {
+
+        return String.join("\n",
+                           streamAllEntries().map(EntryData::toString)
+                                   .toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry),
+                             entry.getChildrenList()
+                                     .stream()
+                                     .flatMap(DocumentStructureWrapper::flatten));
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/LayoutEngineProto.java

@@ -0,0 +1,176 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: LayoutEngine.proto
+// Protobuf Java Version: 4.28.3
+
+package com.iqser.red.service.redaction.v1.server.data;
+
+public final class LayoutEngineProto {
+  private LayoutEngineProto() {}
+  static {
+    com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+      com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+      /* major= */ 4,
+      /* minor= */ 28,
+      /* patch= */ 3,
+      /* suffix= */ "",
+      LayoutEngineProto.class.getName());
+  }
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistryLite registry) {
+  }
+
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+    registerAllExtensions(
+        (com.google.protobuf.ExtensionRegistryLite) registry);
+  }
+  /**
+   * Protobuf enum {@code LayoutEngine}
+   */
+  public enum LayoutEngine
+      implements com.google.protobuf.ProtocolMessageEnum {
+    /**
+     * <code>ALGORITHM = 0;</code>
+     */
+    ALGORITHM(0),
+    /**
+     * <code>AI = 1;</code>
+     */
+    AI(1),
+    /**
+     * <code>OUTLINE = 2;</code>
+     */
+    OUTLINE(2),
+    UNRECOGNIZED(-1),
+    ;
+
+    static {
+      com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+        com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+        /* major= */ 4,
+        /* minor= */ 28,
+        /* patch= */ 3,
+        /* suffix= */ "",
+        LayoutEngine.class.getName());
+    }
+    /**
+     * <code>ALGORITHM = 0;</code>
+     */
+    public static final int ALGORITHM_VALUE = 0;
+    /**
+     * <code>AI = 1;</code>
+     */
+    public static final int AI_VALUE = 1;
+    /**
+     * <code>OUTLINE = 2;</code>
+     */
+    public static final int OUTLINE_VALUE = 2;
+
+
+    public final int getNumber() {
+      if (this == UNRECOGNIZED) {
+        throw new java.lang.IllegalArgumentException(
+            "Can't get the number of an unknown enum value.");
+      }
+      return value;
+    }
+
+    /**
+     * @param value The numeric wire value of the corresponding enum entry.
+     * @return The enum associated with the given numeric wire value.
+     * @deprecated Use {@link #forNumber(int)} instead.
+     */
+    @java.lang.Deprecated
+    public static LayoutEngine valueOf(int value) {
+      return forNumber(value);
+    }
+
+    /**
+     * @param value The numeric wire value of the corresponding enum entry.
+     * @return The enum associated with the given numeric wire value.
+     */
+    public static LayoutEngine forNumber(int value) {
+      switch (value) {
+        case 0: return ALGORITHM;
+        case 1: return AI;
+        case 2: return OUTLINE;
+        default: return null;
+      }
+    }
+
+    public static com.google.protobuf.Internal.EnumLiteMap<LayoutEngine>
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static final com.google.protobuf.Internal.EnumLiteMap<
+        LayoutEngine> internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap<LayoutEngine>() {
+            public LayoutEngine findValueByNumber(int number) {
+              return LayoutEngine.forNumber(number);
+            }
+          };
+
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      if (this == UNRECOGNIZED) {
+        throw new java.lang.IllegalStateException(
+            "Can't get the descriptor of an unrecognized enum value.");
+      }
+      return getDescriptor().getValues().get(ordinal());
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.getDescriptor().getEnumTypes().get(0);
+    }
+
+    private static final LayoutEngine[] VALUES = values();
+
+    public static LayoutEngine valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      if (desc.getIndex() == -1) {
+        return UNRECOGNIZED;
+      }
+      return VALUES[desc.getIndex()];
+    }
+
+    private final int value;
+
+    private LayoutEngine(int value) {
+      this.value = value;
+    }
+
+    // @@protoc_insertion_point(enum_scope:LayoutEngine)
+  }
+
+
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static  com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\022LayoutEngine.proto*2\n\014LayoutEngine\022\r\n\t" +
+      "ALGORITHM\020\000\022\006\n\002AI\020\001\022\013\n\007OUTLINE\020\002BC\n.com." +
+      "iqser.red.service.redaction.v1.server.da" +
+      "taB\021LayoutEngineProtob\006proto3"
+    };
+    descriptor = com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        });
+    descriptor.resolveAllFeaturesImmutable();
+  }
+
+  // @@protoc_insertion_point(outer_class_scope)
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/NodeTypeProto.java

@@ -0,0 +1,261 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: NodeType.proto
+// Protobuf Java Version: 4.28.3
+
+package com.iqser.red.service.redaction.v1.server.data;
+
+public final class NodeTypeProto {
+  private NodeTypeProto() {}
+  static {
+    com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+      com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+      /* major= */ 4,
+      /* minor= */ 28,
+      /* patch= */ 3,
+      /* suffix= */ "",
+      NodeTypeProto.class.getName());
+  }
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistryLite registry) {
+  }
+
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+    registerAllExtensions(
+        (com.google.protobuf.ExtensionRegistryLite) registry);
+  }
+  /**
+   * Protobuf enum {@code NodeType}
+   */
+  public enum NodeType
+      implements com.google.protobuf.ProtocolMessageEnum {
+    /**
+     * <code>DOCUMENT = 0;</code>
+     */
+    DOCUMENT(0),
+    /**
+     * <code>SECTION = 1;</code>
+     */
+    SECTION(1),
+    /**
+     * <code>SUPER_SECTION = 2;</code>
+     */
+    SUPER_SECTION(2),
+    /**
+     * <code>HEADLINE = 3;</code>
+     */
+    HEADLINE(3),
+    /**
+     * <code>PARAGRAPH = 4;</code>
+     */
+    PARAGRAPH(4),
+    /**
+     * <code>TABLE = 5;</code>
+     */
+    TABLE(5),
+    /**
+     * <code>TABLE_CELL = 6;</code>
+     */
+    TABLE_CELL(6),
+    /**
+     * <code>IMAGE = 7;</code>
+     */
+    IMAGE(7),
+    /**
+     * <code>HEADER = 8;</code>
+     */
+    HEADER(8),
+    /**
+     * <code>FOOTER = 9;</code>
+     */
+    FOOTER(9),
+    /**
+     * <code>TABLE_OF_CONTENTS = 10;</code>
+     */
+    TABLE_OF_CONTENTS(10),
+    /**
+     * <code>TABLE_OF_CONTENTS_ITEM = 11;</code>
+     */
+    TABLE_OF_CONTENTS_ITEM(11),
+    UNRECOGNIZED(-1),
+    ;
+
+    static {
+      com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+        com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+        /* major= */ 4,
+        /* minor= */ 28,
+        /* patch= */ 3,
+        /* suffix= */ "",
+        NodeType.class.getName());
+    }
+    /**
+     * <code>DOCUMENT = 0;</code>
+     */
+    public static final int DOCUMENT_VALUE = 0;
+    /**
+     * <code>SECTION = 1;</code>
+     */
+    public static final int SECTION_VALUE = 1;
+    /**
+     * <code>SUPER_SECTION = 2;</code>
+     */
+    public static final int SUPER_SECTION_VALUE = 2;
+    /**
+     * <code>HEADLINE = 3;</code>
+     */
+    public static final int HEADLINE_VALUE = 3;
+    /**
+     * <code>PARAGRAPH = 4;</code>
+     */
+    public static final int PARAGRAPH_VALUE = 4;
+    /**
+     * <code>TABLE = 5;</code>
+     */
+    public static final int TABLE_VALUE = 5;
+    /**
+     * <code>TABLE_CELL = 6;</code>
+     */
+    public static final int TABLE_CELL_VALUE = 6;
+    /**
+     * <code>IMAGE = 7;</code>
+     */
+    public static final int IMAGE_VALUE = 7;
+    /**
+     * <code>HEADER = 8;</code>
+     */
+    public static final int HEADER_VALUE = 8;
+    /**
+     * <code>FOOTER = 9;</code>
+     */
+    public static final int FOOTER_VALUE = 9;
+    /**
+     * <code>TABLE_OF_CONTENTS = 10;</code>
+     */
+    public static final int TABLE_OF_CONTENTS_VALUE = 10;
+    /**
+     * <code>TABLE_OF_CONTENTS_ITEM = 11;</code>
+     */
+    public static final int TABLE_OF_CONTENTS_ITEM_VALUE = 11;
+
+
+    public final int getNumber() {
+      if (this == UNRECOGNIZED) {
+        throw new java.lang.IllegalArgumentException(
+            "Can't get the number of an unknown enum value.");
+      }
+      return value;
+    }
+
+    /**
+     * @param value The numeric wire value of the corresponding enum entry.
+     * @return The enum associated with the given numeric wire value.
+     * @deprecated Use {@link #forNumber(int)} instead.
+     */
+    @java.lang.Deprecated
+    public static NodeType valueOf(int value) {
+      return forNumber(value);
+    }
+
+    /**
+     * @param value The numeric wire value of the corresponding enum entry.
+     * @return The enum associated with the given numeric wire value.
+     */
+    public static NodeType forNumber(int value) {
+      switch (value) {
+        case 0: return DOCUMENT;
+        case 1: return SECTION;
+        case 2: return SUPER_SECTION;
+        case 3: return HEADLINE;
+        case 4: return PARAGRAPH;
+        case 5: return TABLE;
+        case 6: return TABLE_CELL;
+        case 7: return IMAGE;
+        case 8: return HEADER;
+        case 9: return FOOTER;
+        case 10: return TABLE_OF_CONTENTS;
+        case 11: return TABLE_OF_CONTENTS_ITEM;
+        default: return null;
+      }
+    }
+
+    public static com.google.protobuf.Internal.EnumLiteMap<NodeType>
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static final com.google.protobuf.Internal.EnumLiteMap<
+        NodeType> internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap<NodeType>() {
+            public NodeType findValueByNumber(int number) {
+              return NodeType.forNumber(number);
+            }
+          };
+
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      if (this == UNRECOGNIZED) {
+        throw new java.lang.IllegalStateException(
+            "Can't get the descriptor of an unrecognized enum value.");
+      }
+      return getDescriptor().getValues().get(ordinal());
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.iqser.red.service.redaction.v1.server.data.NodeTypeProto.getDescriptor().getEnumTypes().get(0);
+    }
+
+    private static final NodeType[] VALUES = values();
+
+    public static NodeType valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      if (desc.getIndex() == -1) {
+        return UNRECOGNIZED;
+      }
+      return VALUES[desc.getIndex()];
+    }
+
+    private final int value;
+
+    private NodeType(int value) {
+      this.value = value;
+    }
+
+    // @@protoc_insertion_point(enum_scope:NodeType)
+  }
+
+
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static  com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\016NodeType.proto*\306\001\n\010NodeType\022\014\n\010DOCUMEN" +
+      "T\020\000\022\013\n\007SECTION\020\001\022\021\n\rSUPER_SECTION\020\002\022\014\n\010H" +
+      "EADLINE\020\003\022\r\n\tPARAGRAPH\020\004\022\t\n\005TABLE\020\005\022\016\n\nT" +
+      "ABLE_CELL\020\006\022\t\n\005IMAGE\020\007\022\n\n\006HEADER\020\010\022\n\n\006FO" +
+      "OTER\020\t\022\025\n\021TABLE_OF_CONTENTS\020\n\022\032\n\026TABLE_O" +
+      "F_CONTENTS_ITEM\020\013B?\n.com.iqser.red.servi" +
+      "ce.redaction.v1.server.dataB\rNodeTypePro" +
+      "tob\006proto3"
+    };
+    descriptor = com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        });
+    descriptor.resolveAllFeaturesImmutable();
+  }
+
+  // @@protoc_insertion_point(outer_class_scope)
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/RangeProto.java

@@ -0,0 +1,606 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: Range.proto
+// Protobuf Java Version: 4.28.3
+
+package com.iqser.red.service.redaction.v1.server.data;
+
+public final class RangeProto {
+  private RangeProto() {}
+  static {
+    com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+      com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+      /* major= */ 4,
+      /* minor= */ 28,
+      /* patch= */ 3,
+      /* suffix= */ "",
+      RangeProto.class.getName());
+  }
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistryLite registry) {
+  }
+
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+    registerAllExtensions(
+        (com.google.protobuf.ExtensionRegistryLite) registry);
+  }
+  public interface RangeOrBuilder extends
+      // @@protoc_insertion_point(interface_extends:Range)
+      com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * <pre>
+     * A start index.
+     * </pre>
+     *
+     * <code>int32 start = 1;</code>
+     * @return The start.
+     */
+    int getStart();
+
+    /**
+     * <pre>
+     * An end index.
+     * </pre>
+     *
+     * <code>int32 end = 2;</code>
+     * @return The end.
+     */
+    int getEnd();
+  }
+  /**
+   * Protobuf type {@code Range}
+   */
+  public static final class Range extends
+      com.google.protobuf.GeneratedMessage implements
+      // @@protoc_insertion_point(message_implements:Range)
+      RangeOrBuilder {
+  private static final long serialVersionUID = 0L;
+    static {
+      com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(
+        com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+        /* major= */ 4,
+        /* minor= */ 28,
+        /* patch= */ 3,
+        /* suffix= */ "",
+        Range.class.getName());
+    }
+    // Use Range.newBuilder() to construct.
+    private Range(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+      super(builder);
+    }
+    private Range() {
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.iqser.red.service.redaction.v1.server.data.RangeProto.internal_static_Range_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.iqser.red.service.redaction.v1.server.data.RangeProto.internal_static_Range_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.class, com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.Builder.class);
+    }
+
+    public static final int START_FIELD_NUMBER = 1;
+    private int start_ = 0;
+    /**
+     * <pre>
+     * A start index.
+     * </pre>
+     *
+     * <code>int32 start = 1;</code>
+     * @return The start.
+     */
+    @java.lang.Override
+    public int getStart() {
+      return start_;
+    }
+
+    public static final int END_FIELD_NUMBER = 2;
+    private int end_ = 0;
+    /**
+     * <pre>
+     * An end index.
+     * </pre>
+     *
+     * <code>int32 end = 2;</code>
+     * @return The end.
+     */
+    @java.lang.Override
+    public int getEnd() {
+      return end_;
+    }
+
+    private byte memoizedIsInitialized = -1;
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      if (start_ != 0) {
+        output.writeInt32(1, start_);
+      }
+      if (end_ != 0) {
+        output.writeInt32(2, end_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (start_ != 0) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, start_);
+      }
+      if (end_ != 0) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(2, end_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof com.iqser.red.service.redaction.v1.server.data.RangeProto.Range)) {
+        return super.equals(obj);
+      }
+      com.iqser.red.service.redaction.v1.server.data.RangeProto.Range other = (com.iqser.red.service.redaction.v1.server.data.RangeProto.Range) obj;
+
+      if (getStart()
+          != other.getStart()) return false;
+      if (getEnd()
+          != other.getEnd()) return false;
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      hash = (37 * hash) + START_FIELD_NUMBER;
+      hash = (53 * hash) + getStart();
+      hash = (37 * hash) + END_FIELD_NUMBER;
+      hash = (53 * hash) + getEnd();
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        java.nio.ByteBuffer data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        java.nio.ByteBuffer data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input);
+    }
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessage
+          .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+    public static Builder newBuilder(com.iqser.red.service.redaction.v1.server.data.RangeProto.Range prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE
+          ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /**
+     * Protobuf type {@code Range}
+     */
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder> implements
+        // @@protoc_insertion_point(builder_implements:Range)
+        com.iqser.red.service.redaction.v1.server.data.RangeProto.RangeOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.iqser.red.service.redaction.v1.server.data.RangeProto.internal_static_Range_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.iqser.red.service.redaction.v1.server.data.RangeProto.internal_static_Range_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.class, com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.Builder.class);
+      }
+
+      // Construct using com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.newBuilder()
+      private Builder() {
+
+      }
+
+      private Builder(
+          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+        super(parent);
+
+      }
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        start_ = 0;
+        end_ = 0;
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.iqser.red.service.redaction.v1.server.data.RangeProto.internal_static_Range_descriptor;
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.RangeProto.Range getDefaultInstanceForType() {
+        return com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.RangeProto.Range build() {
+        com.iqser.red.service.redaction.v1.server.data.RangeProto.Range result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public com.iqser.red.service.redaction.v1.server.data.RangeProto.Range buildPartial() {
+        com.iqser.red.service.redaction.v1.server.data.RangeProto.Range result = new com.iqser.red.service.redaction.v1.server.data.RangeProto.Range(this);
+        if (bitField0_ != 0) { buildPartial0(result); }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartial0(com.iqser.red.service.redaction.v1.server.data.RangeProto.Range result) {
+        int from_bitField0_ = bitField0_;
+        if (((from_bitField0_ & 0x00000001) != 0)) {
+          result.start_ = start_;
+        }
+        if (((from_bitField0_ & 0x00000002) != 0)) {
+          result.end_ = end_;
+        }
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.iqser.red.service.redaction.v1.server.data.RangeProto.Range) {
+          return mergeFrom((com.iqser.red.service.redaction.v1.server.data.RangeProto.Range)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(com.iqser.red.service.redaction.v1.server.data.RangeProto.Range other) {
+        if (other == com.iqser.red.service.redaction.v1.server.data.RangeProto.Range.getDefaultInstance()) return this;
+        if (other.getStart() != 0) {
+          setStart(other.getStart());
+        }
+        if (other.getEnd() != 0) {
+          setEnd(other.getEnd());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 8: {
+                start_ = input.readInt32();
+                bitField0_ |= 0x00000001;
+                break;
+              } // case 8
+              case 16: {
+                end_ = input.readInt32();
+                bitField0_ |= 0x00000002;
+                break;
+              } // case 16
+              default: {
+                if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                  done = true; // was an endgroup tag
+                }
+                break;
+              } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+      private int bitField0_;
+
+      private int start_ ;
+      /**
+       * <pre>
+       * A start index.
+       * </pre>
+       *
+       * <code>int32 start = 1;</code>
+       * @return The start.
+       */
+      @java.lang.Override
+      public int getStart() {
+        return start_;
+      }
+      /**
+       * <pre>
+       * A start index.
+       * </pre>
+       *
+       * <code>int32 start = 1;</code>
+       * @param value The start to set.
+       * @return This builder for chaining.
+       */
+      public Builder setStart(int value) {
+
+        start_ = value;
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return this;
+      }
+      /**
+       * <pre>
+       * A start index.
+       * </pre>
+       *
+       * <code>int32 start = 1;</code>
+       * @return This builder for chaining.
+       */
+      public Builder clearStart() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        start_ = 0;
+        onChanged();
+        return this;
+      }
+
+      private int end_ ;
+      /**
+       * <pre>
+       * An end index.
+       * </pre>
+       *
+       * <code>int32 end = 2;</code>
+       * @return The end.
+       */
+      @java.lang.Override
+      public int getEnd() {
+        return end_;
+      }
+      /**
+       * <pre>
+       * An end index.
+       * </pre>
+       *
+       * <code>int32 end = 2;</code>
+       * @param value The end to set.
+       * @return This builder for chaining.
+       */
+      public Builder setEnd(int value) {
+
+        end_ = value;
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /**
+       * <pre>
+       * An end index.
+       * </pre>
+       *
+       * <code>int32 end = 2;</code>
+       * @return This builder for chaining.
+       */
+      public Builder clearEnd() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        end_ = 0;
+        onChanged();
+        return this;
+      }
+
+      // @@protoc_insertion_point(builder_scope:Range)
+    }
+
+    // @@protoc_insertion_point(class_scope:Range)
+    private static final com.iqser.red.service.redaction.v1.server.data.RangeProto.Range DEFAULT_INSTANCE;
+    static {
+      DEFAULT_INSTANCE = new com.iqser.red.service.redaction.v1.server.data.RangeProto.Range();
+    }
+
+    public static com.iqser.red.service.redaction.v1.server.data.RangeProto.Range getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser<Range>
+        PARSER = new com.google.protobuf.AbstractParser<Range>() {
+      @java.lang.Override
+      public Range parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        Builder builder = newBuilder();
+        try {
+          builder.mergeFrom(input, extensionRegistry);
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.setUnfinishedMessage(builder.buildPartial());
+        } catch (com.google.protobuf.UninitializedMessageException e) {
+          throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial());
+        } catch (java.io.IOException e) {
+          throw new com.google.protobuf.InvalidProtocolBufferException(e)
+              .setUnfinishedMessage(builder.buildPartial());
+        }
+        return builder.buildPartial();
+      }
+    };
+
+    public static com.google.protobuf.Parser<Range> parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<Range> getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.iqser.red.service.redaction.v1.server.data.RangeProto.Range getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+
+  }
+
+  private static final com.google.protobuf.Descriptors.Descriptor
+    internal_static_Range_descriptor;
+  private static final 
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_Range_fieldAccessorTable;
+
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static  com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\013Range.proto\"#\n\005Range\022\r\n\005start\030\001 \001(\005\022\013\n" +
+      "\003end\030\002 \001(\005B<\n.com.iqser.red.service.reda" +
+      "ction.v1.server.dataB\nRangeProtob\006proto3"
+    };
+    descriptor = com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        });
+    internal_static_Range_descriptor =
+      getDescriptor().getMessageTypes().get(0);
+    internal_static_Range_fieldAccessorTable = new
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+        internal_static_Range_descriptor,
+        new java.lang.String[] { "Start", "End", });
+    descriptor.resolveAllFeaturesImmutable();
+  }
+
+  // @@protoc_insertion_point(outer_class_scope)
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPage.java

@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+import java.io.Serializable;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Deprecated
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPage implements Serializable {
+
+    int number;
+    int height;
+    int width;
+    int rotation;
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentPositionData.java

@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+import java.io.Serializable;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Deprecated
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPositionData implements Serializable {
+
+    Long id;
+    int[] stringIdxToPositionIdx;
+    float[][] positions;
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentStructure.java

@@ -0,0 +1,158 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+import java.awt.geom.Rectangle2D;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Deprecated
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentStructure implements Serializable {
+
+    EntryData root;
+
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+        public static final String REPRESENTATION_HASH = "representationHash";
+
+    }
+
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public static double[] parseRepresentationVector(String representationHash) {
+
+        String[] stringArray = representationHash.split("[,\\s]+");
+        double[] doubleArray = new double[stringArray.length];
+        for (int i = 0; i < stringArray.length; i++) {
+            doubleArray[i] = Double.parseDouble(stringArray[i]);
+        }
+
+        return doubleArray;
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return root;
+        }
+        EntryData entry = root.children.get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.children.get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return Stream.concat(Stream.of(root), root.children.stream())
+                .flatMap(DocumentStructure::flatten);
+    }
+
+
+    public String toString() {
+
+        return String.join("\n",
+                           streamAllEntries().map(EntryData::toString)
+                                   .toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry),
+                             entry.children.stream()
+                                     .flatMap(DocumentStructure::flatten));
+    }
+
+
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    public static class EntryData implements Serializable {
+
+        NodeType type;
+        int[] treeId;
+        Long[] atomicBlockIds;
+        Long[] pageNumbers;
+        Map<String, String> properties;
+        List<EntryData> children;
+        Set<LayoutEngine> engines;
+
+
+        @Override
+        public String toString() {
+
+            StringBuilder sb = new StringBuilder();
+            sb.append("[");
+            for (int i : treeId) {
+                sb.append(i);
+                sb.append(",");
+            }
+            sb.delete(sb.length() - 1, sb.length());
+            sb.append("]: ");
+
+            sb.append(type);
+            sb.append(" atbs = ");
+            sb.append(atomicBlockIds.length);
+
+            return sb.toString();
+        }
+
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/DocumentTextData.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+import java.io.Serializable;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Deprecated
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentTextData implements Serializable {
+
+    Long id;
+    Long page;
+    String searchText;
+    int numberOnPage;
+    int start;
+    int end;
+    int[] lineBreaks;
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/LayoutEngine.java

@@ -0,0 +1,8 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+@Deprecated
+public enum LayoutEngine {
+    ALGORITHM,
+    AI,
+    OUTLINE
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/data/old/NodeType.java

@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.data.old;
+
+import java.io.Serializable;
+import java.util.Locale;
+
+@Deprecated
+public enum NodeType implements Serializable {
+    DOCUMENT,
+    SECTION,
+    SUPER_SECTION,
+    HEADLINE,
+    PARAGRAPH,
+    TABLE,
+    TABLE_CELL,
+    IMAGE,
+    HEADER,
+    FOOTER;
+
+
+    public String toString() {
+
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+    }
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java

@@ -0,0 +1,199 @@
+package com.iqser.red.service.redaction.v1.server.mapper;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPageProto.AllDocumentPages;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPageProto.DocumentPage;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.AllDocumentPositionData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.DocumentPositionData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.DocumentPositionData.Position;
+import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto.DocumentStructure;
+import com.iqser.red.service.redaction.v1.server.data.DocumentStructureWrapper;
+import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto.AllDocumentTextData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto.DocumentTextData;
+import com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData;
+import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto;
+import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto;
+import com.iqser.red.service.redaction.v1.server.data.RangeProto;
+import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class DocumentDataMapper {
+
+    public DocumentData toDocumentData(Document document) {
+
+        List<DocumentTextData> documentTextData = document.streamTerminalTextBlocksInOrder()
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks()
+                        .stream())
+                .distinct()
+                .map(DocumentDataMapper::toAtomicTextBlockData)
+                .toList();
+
+        AllDocumentTextData allDocumentTextData = AllDocumentTextData.newBuilder().addAllDocumentTextData(documentTextData).build();
+
+        List<DocumentPositionData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks()
+                        .stream())
+                .distinct()
+                .map(DocumentDataMapper::toAtomicPositionBlockData)
+                .toList();
+
+        AllDocumentPositionData allDocumentPositionData = AllDocumentPositionData.newBuilder().addAllDocumentPositionData(atomicPositionBlockData).build();
+
+        List<DocumentPage> documentPageData = document.getPages()
+                .stream()
+                .sorted(Comparator.comparingInt(Page::getNumber))
+                .map(DocumentDataMapper::toPageData)
+                .toList();
+
+        AllDocumentPages allDocumentPages = AllDocumentPages.newBuilder().addAllDocumentPages(documentPageData).build();
+        DocumentStructureWrapper tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
+        return DocumentData.builder()
+                .documentTextData(allDocumentTextData)
+                .documentPositionData(allDocumentPositionData)
+                .documentPages(allDocumentPages)
+                .documentStructureWrapper(tableOfContentsData)
+                .build();
+    }
+
+
+    private DocumentStructureWrapper toDocumentTreeData(DocumentTree documentTree) {
+
+        return new DocumentStructureWrapper(DocumentStructure.newBuilder().setRoot(toEntryData(documentTree.getRoot())).build());
+    }
+
+
+    private EntryData toEntryData(DocumentTree.Entry entry) {
+
+        List<Long> atomicTextBlocks;
+
+        if (entry.getNode().isLeaf()) {
+            atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getLeafTextBlock());
+        } else {
+            atomicTextBlocks = new ArrayList<>();
+        }
+
+        Map<String, String> properties = switch (entry.getType()) {
+            case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
+            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
+            case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
+            case PARAGRAPH ->
+                    entry.getNode() instanceof DuplicatedParagraph duplicatedParagraph ? PropertiesMapper.buildDuplicateParagraphProperties(duplicatedParagraph) : new HashMap<>();
+            default -> new HashMap<>();
+        };
+
+        var documentBuilder = EntryData.newBuilder()
+                .addAllTreeId(entry.getTreeId())
+                .addAllChildren(entry.getChildren()
+                                        .stream()
+                                        .map(DocumentDataMapper::toEntryData)
+                                        .toList())
+                .setType(resolveType(entry.getType()))
+                .addAllAtomicBlockIds(atomicTextBlocks)
+                .addAllPageNumbers(entry.getNode().getPages()
+                                           .stream()
+                                           .map(Page::getNumber)
+                                           .map(Integer::longValue)
+                                           .toList())
+                .putAllProperties(properties);
+
+        if (entry.getNode() != null) {
+            documentBuilder.addAllEngines(entry.getNode().getEngines()
+                                                  .stream()
+                                                  .map(engine -> LayoutEngineProto.LayoutEngine.valueOf(engine.name()))
+                                                  .toList());
+        } else {
+            documentBuilder.addAllEngines(new HashSet<>(Set.of(LayoutEngineProto.LayoutEngine.ALGORITHM)));
+        }
+        return documentBuilder.build();
+    }
+
+
+    private static NodeTypeProto.NodeType resolveType(NodeType type) {
+
+        return NodeTypeProto.NodeType.valueOf(type.name());
+    }
+
+
+    private List<Long> toAtomicTextBlockIds(TextBlock textBlock) {
+
+        return textBlock.getAtomicTextBlocks()
+                .stream()
+                .map(AtomicTextBlock::getId)
+                .toList();
+    }
+
+
+    private DocumentPage toPageData(Page p) {
+
+        return DocumentPage.newBuilder().setRotation(p.getRotation()).setHeight(p.getHeight()).setWidth(p.getWidth()).setNumber(p.getNumber()).build();
+    }
+
+
+    private DocumentTextData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
+
+        return DocumentTextData.newBuilder()
+                .setId(atomicTextBlock.getId())
+                .setPage(atomicTextBlock.getPage().getNumber().longValue())
+                .setSearchText(atomicTextBlock.getSearchText())
+                .setNumberOnPage(atomicTextBlock.getNumberOnPage())
+                .setStart(atomicTextBlock.getTextRange().start())
+                .setEnd(atomicTextBlock.getTextRange().end())
+                .addAllLineBreaks(atomicTextBlock.getLineBreaks())
+                .addAllItalicTextRanges(atomicTextBlock.getItalicTextRanges()
+                                                .stream()
+                                                .map(r -> RangeProto.Range.newBuilder().setStart(r.start()).setEnd(r.end()).build())
+                                                .toList())
+                .addAllBoldTextRanges(atomicTextBlock.getBoldTextRanges()
+                                              .stream()
+                                              .map(r -> RangeProto.Range.newBuilder().setStart(r.start()).setEnd(r.end()).build())
+                                              .toList())
+                .build();
+    }
+
+
+    private DocumentPositionData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
+
+        return DocumentPositionData.newBuilder()
+                .setId(atomicTextBlock.getId())
+                .addAllPositions(toPositions(atomicTextBlock.getPositions()))
+                .addAllStringIdxToPositionIdx(atomicTextBlock.getStringIdxToPositionIdx())
+                .build();
+    }
+
+
+    private static List<Position> toPositions(List<Rectangle2D> rects) {
+
+        List<Position> positions = new ArrayList<>();
+        for (Rectangle2D rect : rects) {
+            positions.add(toPosition(rect));
+        }
+        return positions;
+    }
+
+
+    private static Position toPosition(Rectangle2D rect) {
+
+        return Position.newBuilder().addValue((float) rect.getMinX()).addValue((float) rect.getMinY()).addValue((float) rect.getWidth()).addValue((float) rect.getHeight()).build();
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java

@@ -1,14 +1,18 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.mapper;
+
+import static com.iqser.red.service.redaction.v1.server.data.DocumentPageProto.DocumentPage;
+import static com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.AllDocumentPositionData;
+import static com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto.AllDocumentTextData;
+import static com.iqser.red.service.redaction.v1.server.data.EntryDataProto.EntryData;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
+import com.iqser.red.service.redaction.v1.server.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
@@ -16,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
@@ -23,13 +28,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContentsItem;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 
 import lombok.experimental.UtilityClass;
 
@@ -42,47 +45,45 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
-                                        .map(DocumentGraphMapper::buildPage)
-                                        .toList());
-
-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildrenList(), context));
 
         document.setDocumentTree(context.documentTree);
         document.setPages(new HashSet<>(context.pageData));
-        document.setNumberOfPages(documentData.getDocumentPages().length);
+        document.setNumberOfPages(documentData.getDocumentPages().getDocumentPagesCount());
 
         document.setTextBlock(document.getTextBlock());
         return document;
     }
 
 
-    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
+    private List<DocumentTree.Entry> buildEntries(List<EntryData> entries, Context context) {
 
         List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size());
-        for (DocumentStructure.EntryData entryData : entries) {
+        for (EntryData entryData : entries) {
 
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
+            List<Page> pages = entryData.getPageNumbersList()
+                    .stream()
                     .map(context::getPage)
                     .toList();
 
             SemanticNode node = switch (entryData.getType()) {
                 case SECTION -> buildSection(context);
                 case SUPER_SECTION -> buildSuperSection(context);
-                case PARAGRAPH -> buildParagraph(context, entryData.getProperties());
+                case PARAGRAPH -> buildParagraph(context, entryData.getPropertiesMap());
                 case HEADLINE -> buildHeadline(context);
                 case HEADER -> buildHeader(context);
                 case FOOTER -> buildFooter(context);
-                case TABLE -> buildTable(context, entryData.getProperties());
-                case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
+                case TABLE -> buildTable(context, entryData.getPropertiesMap());
+                case TABLE_CELL -> buildTableCell(context, entryData.getPropertiesMap());
+                case IMAGE -> buildImage(context, entryData.getPropertiesMap(), entryData.getPageNumbersList());
+                case TABLE_OF_CONTENTS -> buildTableOfContents(context);
+                case TABLE_OF_CONTENTS_ITEM -> buildTableOfContentsItem(context);
                 default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
             };
 
-            if (entryData.getAtomicBlockIds().length > 0) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
+            if (entryData.getAtomicBlockIdsCount() > 0) {
+                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIdsList(), context, node);
                 node.setLeafTextBlock(textBlock);
-
                 switch (entryData.getType()) {
                     case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
                     case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
@@ -90,35 +91,43 @@ public class DocumentGraphMapper {
                     default -> textBlock.getAtomicTextBlocks()
                             .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
                 }
-
-            }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
-                    .toList();
-            if (entryData.getEngines() != null) {
-                entryData.getEngines()
-                        .forEach(node::addEngine);
-            } else {
-                entryData.setEngines(Collections.emptySet());
             }
+            List<Integer> treeId = entryData.getTreeIdList();
             node.setTreeId(treeId);
 
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
+            entryData.getEnginesList()
+                    .stream()
+                    .map(engine -> LayoutEngine.valueOf(engine.name()))
+                    .forEach(node::addEngine);
 
+            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
         }
         return newEntries;
     }
 
 
+    private static TableOfContents buildTableOfContents(Context context) {
+
+        return TableOfContents.builder().documentTree(context.documentTree).build();
+    }
+
+
+    private static TableOfContentsItem buildTableOfContentsItem(Context context) {
+
+        return TableOfContentsItem.builder().documentTree(context.documentTree).build();
+    }
+
+
     private Headline buildHeadline(Context context) {
 
         return Headline.builder().documentTree(context.documentTree).build();
     }
 
 
-    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
+    private Image buildImage(Context context, Map<String, String> properties, List<Long> pageNumbers) {
 
-        assert pageNumbers.length == 1;
-        Page page = context.getPage(pageNumbers[0]);
+        assert pageNumbers.size() == 1;
+        Page page = context.getPage(pageNumbers.get(0));
         var builder = Image.builder();
         PropertiesMapper.parseImageProperties(properties, builder);
         return builder.documentTree(context.documentTree).page(page).build();
@@ -171,7 +180,7 @@ public class DocumentGraphMapper {
 
             DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
 
-            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            var unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
             duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
             return duplicatedParagraph;
         }
@@ -180,9 +189,9 @@ public class DocumentGraphMapper {
     }
 
 
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
+    private TextBlock toTextBlock(List<Long> atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return Arrays.stream(atomicTextBlockIds)
+        return atomicTextBlockIds.stream()
                 .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
                 .collect(new TextBlockCollector());
     }
@@ -190,10 +199,10 @@ public class DocumentGraphMapper {
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
-        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
-                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
+        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)),
+                                                       context.documentPositionData.getDocumentPositionData(Math.toIntExact(atomicTextBlockId)),
                                                        parent,
-                                                       context.getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage()));
+                                                       context.getPage(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)).getPage()));
     }
 
 
@@ -207,23 +216,25 @@ public class DocumentGraphMapper {
 
         private final DocumentTree documentTree;
         private final List<Page> pageData;
-        private final List<DocumentTextData> documentTextData;
-        private final List<DocumentPositionData> documentPositionData;
+        private final AllDocumentTextData documentTextData;
+        private final AllDocumentPositionData documentPositionData;
 
 
         Context(DocumentData documentData, DocumentTree documentTree) {
 
             this.documentTree = documentTree;
-            this.pageData = new ArrayList<>();
-            this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
-                    .toList();
-            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
+            this.pageData = documentData.getDocumentPages().getDocumentPagesList()
+                    .stream()
+                    .map(DocumentGraphMapper::buildPage)
+                    .sorted(Comparator.comparingInt(Page::getNumber))
                     .toList();
+            this.documentTextData = documentData.getDocumentTextData();
+            this.documentPositionData = documentData.getDocumentPositionData();
 
         }
 
 
-        private Page getPage(Long pageIndex) {
+        public Page getPage(Long pageIndex) {
 
             Page page = pageData.get(Math.toIntExact(pageIndex) - 1);
             assert page.getNumber() == Math.toIntExact(pageIndex);
@@ -233,3 +244,5 @@ public class DocumentGraphMapper {
     }
 
 }
+
+

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/PropertiesMapper.java

@@ -0,0 +1,152 @@
+package com.iqser.red.service.redaction.v1.server.mapper;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.iqser.red.service.redaction.v1.server.data.DocumentStructureWrapper;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class PropertiesMapper {
+
+    public static Map<String, String> buildImageProperties(Image image) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE, image.getImageType().name());
+        properties.put(DocumentStructureWrapper.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
+        properties.put(DocumentStructureWrapper.ImageProperties.POSITION, toString(image.getPosition()));
+        properties.put(DocumentStructureWrapper.ImageProperties.ID, image.getId());
+        properties.put(DocumentStructureWrapper.ImageProperties.REPRESENTATION_HASH, image.getRepresentationHash());
+        return properties;
+    }
+
+
+    public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(DocumentStructureWrapper.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
+        properties.put(DocumentStructureWrapper.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
+        properties.put(DocumentStructureWrapper.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
+
+        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
+            throw new IllegalArgumentException("TableCell can only occur on a single page!");
+        }
+        String bBoxString = toString(tableCell.getBBox()
+                                             .get(tableCell.getPages()
+                                                          .stream()
+                                                          .findFirst()
+                                                          .get()));
+        properties.put(DocumentStructureWrapper.TableCellProperties.B_BOX, bBoxString);
+
+        return properties;
+    }
+
+
+    public static Map<String, String> buildTableProperties(Table table) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
+        properties.put(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
+        return properties;
+    }
+
+
+    public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder<?, ?> builder) {
+
+        builder.imageType(parseImageType(properties.get(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.ImageProperties.TRANSPARENT)));
+        builder.position(DocumentStructureWrapper.parseRectangle2D(properties.get(DocumentStructureWrapper.ImageProperties.POSITION)));
+        builder.id(properties.get(DocumentStructureWrapper.ImageProperties.ID));
+    }
+
+
+    public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder<?, ?> builder) {
+
+        builder.row(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.ROW)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.COL)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.TableCellProperties.HEADER)));
+        builder.bBox(DocumentStructureWrapper.parseRectangle2D(properties.get(DocumentStructureWrapper.TableCellProperties.B_BOX)));
+    }
+
+
+    public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
+
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS)));
+    }
+
+
+    public static Map<String, String> buildDuplicateParagraphProperties(DuplicatedParagraph duplicatedParagraph) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID,
+                       Arrays.toString(toAtomicTextBlockIds(duplicatedParagraph.getUnsortedLeafTextBlock())));
+        return properties;
+    }
+
+
+    public static boolean isDuplicateParagraph(Map<String, String> properties) {
+
+        return properties.containsKey(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+    }
+
+
+    public static List<Long> getUnsortedTextblockIds(Map<String, String> properties) {
+
+        return toLongList(properties.get(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+    }
+
+
+    public static List<Long> toLongList(String ids) {
+
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(","))
+                .map(Long::valueOf)
+                .toList();
+    }
+
+
+    private static ImageType parseImageType(String imageType) {
+
+        try {
+            return ImageType.valueOf(imageType.toUpperCase(Locale.ROOT));
+        } catch (IllegalArgumentException e) {
+            return ImageType.OTHER;
+        }
+    }
+
+
+    public static String toString(Rectangle2D rectangle2D) {
+
+        return String.format(Locale.US,
+                             "%f%s%f%s%f%s%f",
+                             rectangle2D.getX(),
+                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
+                             rectangle2D.getY(),
+                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
+                             rectangle2D.getWidth(),
+                             DocumentStructureWrapper.RECTANGLE_DELIMITER,
+                             rectangle2D.getHeight());
+    }
+
+
+    private static Long[] toAtomicTextBlockIds(TextBlock textBlock) {
+
+        return textBlock.getAtomicTextBlocks()
+                .stream()
+                .map(AtomicTextBlock::getId)
+                .toArray(Long[]::new);
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/AbstractNodeVisitor.java

@@ -0,0 +1,116 @@
+package com.iqser.red.service.redaction.v1.server.model.document;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContentsItem;
+
+public abstract class AbstractNodeVisitor implements NodeVisitor {
+
+    @Override
+    public void visit(Document document) {
+
+        defaultVisit(document);
+    }
+
+
+    @Override
+    public void visit(SuperSection superSection) {
+
+        defaultVisit(superSection);
+    }
+
+
+    @Override
+    public void visit(Section section) {
+
+        defaultVisit(section);
+    }
+
+
+    @Override
+    public void visit(Headline headline) {
+
+        defaultVisit(headline);
+    }
+
+
+    @Override
+    public void visit(Paragraph paragraph) {
+
+        defaultVisit(paragraph);
+    }
+
+
+    @Override
+    public void visit(Footer footer) {
+
+        defaultVisit(footer);
+    }
+
+
+    @Override
+    public void visit(Header header) {
+
+        defaultVisit(header);
+    }
+
+
+    @Override
+    public void visit(Image image) {
+
+        defaultVisit(image);
+    }
+
+
+    @Override
+    public void visit(Table table) {
+
+        defaultVisit(table);
+    }
+
+
+    @Override
+    public void visit(TableCell tableCell) {
+
+        defaultVisit(tableCell);
+    }
+
+
+    @Override
+    public void visit(TableOfContents toc) {
+
+        defaultVisit(toc);
+    }
+
+
+    @Override
+    public void visit(TableOfContentsItem toci) {
+
+        defaultVisit(toci);
+    }
+
+
+    public void visitNodeDefault(SemanticNode node) {
+        // By default, it does nothing
+    }
+
+
+    protected void defaultVisit(SemanticNode semanticNode) {
+
+        visitNodeDefault(semanticNode);
+        semanticNode.streamChildren()
+                .forEach(node -> node.accept(this));
+
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -9,6 +9,8 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
 
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
@@ -17,6 +19,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.utils.EntityCreationUtility;
+import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -35,7 +39,7 @@ public class DocumentTree {
 
     public DocumentTree(Document document) {
 
-        root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build();
+        this.root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build();
     }
 
 
@@ -296,6 +300,22 @@ public class DocumentTree {
     }
 
 
+    public Optional<Entry> findEntryById(List<Integer> treeId) {
+
+        if (treeId.isEmpty()) {
+            return Optional.of(root);
+        }
+        Entry entry = root;
+        for (int id : treeId) {
+            if (id < 0 || id >= entry.children.size()) {
+                return Optional.empty();
+            }
+            entry = entry.children.get(id);
+        }
+        return Optional.of(entry);
+    }
+
+
     public Stream<Entry> mainEntries() {
 
         return root.children.stream();
@@ -342,6 +362,25 @@ public class DocumentTree {
     }
 
 
+    public void addEntityToGraph(TextEntity entity) {
+
+        getRoot().getNode().addThisToEntityIfIntersects(entity);
+
+        TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
+        EntityEnrichmentService.enrichEntity(entity, textBlock);
+
+        EntityCreationUtility.addToPages(entity);
+        EntityCreationUtility.addEntityToNodeEntitySets(entity);
+
+        if (entity.getEntityType().equals(EntityType.TEMPORARY)) {
+            return;
+        }
+
+        entity.computeRelations();
+        entity.notifyEntityInserted();
+    }
+
+
     @Builder
     @Getter
     @AllArgsConstructor

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java

@@ -0,0 +1,32 @@
+package com.iqser.red.service.redaction.v1.server.model.document;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+import lombok.Getter;
+
+public class IntersectingNodeVisitor extends AbstractNodeVisitor {
+
+    @Getter
+    private Set<SemanticNode> intersectingNodes;
+    private final TextRange textRange;
+
+
+    public IntersectingNodeVisitor(TextRange textRange) {
+
+        this.textRange = textRange;
+        this.intersectingNodes = new HashSet<>();
+    }
+
+
+    @Override
+    public void visitNodeDefault(SemanticNode node) {
+
+        if (textRange.intersects(node.getTextRange())) {
+            intersectingNodes.add(node);
+        }
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/NodeVisitor.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.model.document;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableOfContentsItem;
+
+public interface NodeVisitor {
+
+    void visit(Document document);
+
+
+    void visit(SuperSection superSection);
+
+
+    void visit(Section section);
+
+
+    void visit(Headline headline);
+
+
+    void visit(Paragraph paragraph);
+
+
+    void visit(Footer footer);
+
+
+    void visit(Header header);
+
+
+    void visit(Image image);
+
+
+    void visit(Table table);
+
+
+    void visit(TableCell tableCell);
+
+
+    void visit(TableOfContents tableOfContents);
+
+
+    void visit(TableOfContentsItem tableOfContentsItem);
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java

@@ -134,6 +134,12 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    public boolean containsExclusive(int index) {
+
+        return start <= index && index < end;
+    }
+
+
     /**
      * Checks if this {@link TextRange} intersects with another {@link TextRange}.
      *

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@Getter
+@RequiredArgsConstructor
+public abstract class AbstractRelation implements Relation {
+
+    protected final TextEntity a;
+    protected final TextEntity b;
+
+
+    @Override
+    public String toString() {
+
+        return this.getClass().getSimpleName() + "{" + "a=" + a + ", b=" + b + '}';
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+public class Containment extends Intersection {
+
+    public Containment(TextEntity container, TextEntity contained) {
+
+        super(container, contained);
+    }
+
+    public TextEntity getContainer() {
+        return a;
+    }
+
+    public TextEntity getContained() {
+        return b;
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityEventListener.java

@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+public interface EntityEventListener {
+
+    /**
+     * Invoked when an entity is inserted.
+     *
+     * @param entity The entity that was inserted.
+     */
+    void onEntityInserted(IEntity entity);
+
+    /**
+     * Invoked when an entity is updated.
+     *
+     * @param entity The entity that was updated.
+     */
+    void onEntityUpdated(IEntity entity);
+
+    /**
+     * Invoked when an entity is removed.
+     *
+     * @param entity The entity that was removed.
+     */
+    void onEntityRemoved(IEntity entity);
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java

@@ -6,5 +6,6 @@ public enum EntityType {
     RECOMMENDATION,
     FALSE_POSITIVE,
     FALSE_RECOMMENDATION,
-    DICTIONARY_REMOVAL
+    DICTIONARY_REMOVAL,
+    TEMPORARY
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+public class Equality extends Containment {
+
+    public Equality(TextEntity a, TextEntity b) {
+
+        super(a, b);
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -6,7 +6,6 @@ import java.util.PriorityQueue;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
 
 import lombok.NonNull;
 
@@ -96,6 +95,9 @@ public interface IEntity {
     // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
     default boolean applied() {
 
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isApplied();
+        }
         return getManualOverwrite().getApplied()
                 .orElse(getMatchedRule().isApplied());
     }
@@ -119,6 +121,10 @@ public interface IEntity {
      */
     default boolean ignored() {
 
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isIgnored();
+        }
+
         return getManualOverwrite().getIgnored()
                 .orElse(getMatchedRule().isIgnored());
     }
@@ -131,6 +137,9 @@ public interface IEntity {
      */
     default boolean removed() {
 
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isRemoved();
+        }
         return getManualOverwrite().getRemoved()
                 .orElse(getMatchedRule().isRemoved());
     }
@@ -143,6 +152,9 @@ public interface IEntity {
      */
     default boolean resized() {
 
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isRemoved();
+        }
         return getManualOverwrite().getResized()
                 .orElse(false);
     }
@@ -327,7 +339,9 @@ public interface IEntity {
      */
     default void addMatchedRule(MatchedRule matchedRule) {
 
+        boolean wasValid = valid();
         getMatchedRuleList().add(matchedRule);
+        handleStateChange(wasValid);
     }
 
 
@@ -341,7 +355,53 @@ public interface IEntity {
         if (getMatchedRuleList().equals(matchedRules)) {
             return;
         }
+        boolean wasValid = valid();
         getMatchedRuleList().addAll(matchedRules);
+        handleStateChange(wasValid);
+    }
+
+
+    void addEntityEventListener(EntityEventListener listener);
+
+
+    void removeEntityEventListener(EntityEventListener listener);
+
+
+    default void notifyEntityInserted() {
+
+        for (EntityEventListener listener : getEntityEventListeners()) {
+            listener.onEntityInserted(this);
+        }
+    }
+
+    default void notifyEntityUpdated() {
+
+        for (EntityEventListener listener : getEntityEventListeners()) {
+            listener.onEntityUpdated(this);
+        }
+    }
+
+    default void notifyEntityRemoved() {
+
+        for (EntityEventListener listener : getEntityEventListeners()) {
+            listener.onEntityRemoved(this);
+        }
+    }
+
+
+    Collection<EntityEventListener> getEntityEventListeners();
+
+
+    default void handleStateChange(boolean wasValid) {
+
+        if (valid() == wasValid) {
+            return;
+        }
+        if (!removed()) {
+            notifyEntityUpdated();
+        } else {
+            notifyEntityRemoved();
+        }
     }
 
 
@@ -375,15 +435,9 @@ public interface IEntity {
      *
      * @return The built reason string.
      */
-    default String buildReasonWithManualChangeDescriptions() {
+    default String buildReason() {
 
-        if (getManualOverwrite().getDescriptions().isEmpty()) {
-            return getMatchedRule().getReason();
-        }
-        if (getMatchedRule().getReason().isEmpty()) {
-            return String.join(", ", getManualOverwrite().getDescriptions());
-        }
-        return getMatchedRule().getReason() + ", " + String.join(", ", getManualOverwrite().getDescriptions());
+        return getMatchedRule().getReason();
     }
 
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+public class Intersection extends AbstractRelation {
+
+    public Intersection(TextEntity a, TextEntity b) {
+
+        super(a, b);
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -1,10 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
@@ -14,7 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -26,18 +23,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class ManualChangeOverwrite {
 
-    private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-                                                                                                          ManualRedactionEntry.class, "created by manual change", //
-                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
-                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
-                                                                                                          ManualForceRedaction.class, "forced by manual override", //
-                                                                                                          IdRemoval.class, "removed by manual override", //
-                                                                                                          ManualRecategorization.class, "recategorized by manual override");
-
     @Builder.Default
     List<BaseAnnotation> manualChanges = new LinkedList<>();
     boolean changed;
-    List<String> descriptions;
     String type;
     String legalBasis;
     String section;
@@ -63,6 +51,7 @@ public class ManualChangeOverwrite {
         this.manualChanges = new LinkedList<>();
     }
 
+
     public ManualChangeOverwrite(EntityType entityType, String section) {
 
         this(entityType);
@@ -95,8 +84,6 @@ public class ManualChangeOverwrite {
 
     private void updateFields(List<BaseAnnotation> sortedManualChanges) {
 
-        descriptions = new LinkedList<>();
-
         for (BaseAnnotation manualChange : sortedManualChanges) {
             // ManualRedactionEntries are created prior to rule execution in analysis service.
 
@@ -151,8 +138,6 @@ public class ManualChangeOverwrite {
                     legalBasis = recategorization.getLegalBasis();
                 }
             }
-
-            descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
         }
         changed = false;
     }
@@ -245,13 +230,6 @@ public class ManualChangeOverwrite {
     }
 
 
-    public List<String> getDescriptions() {
-
-        calculateCurrentOverride();
-        return descriptions == null ? Collections.emptyList() : descriptions;
-    }
-
-
     public Optional<List<RectangleWithPage>> getPositions() {
 
         calculateCurrentOverride();

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -5,9 +5,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleType;
-
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -28,8 +25,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
     public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
     public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
+    public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
     public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, MANUAL_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default
@@ -57,6 +55,13 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     }
 
 
+    public boolean isHigherPriorityThanManual() {
+
+        return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) && (RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(
+                MANUAL_TYPE));
+    }
+
+
     /**
      * Returns a modified instance of {@link MatchedRule} based on its applied status.
      * If the rule has been applied, it returns a new {@link MatchedRule} instance that retains all properties of the original

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RectangleWithPage.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model;
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.awt.geom.Rectangle2D;
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
+
+public interface Relation {
+
+    TextEntity getA();
+
+
+    TextEntity getB();
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RuleIdentifier.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.util.Objects;
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RuleType.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
+package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.util.regex.Pattern;
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -4,6 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -11,7 +12,10 @@ import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
 
+import org.apache.commons.collections4.map.HashedMap;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@@ -43,13 +47,14 @@ public class TextEntity implements IEntity {
 
     TextRange textRange;
     @Builder.Default
-    List<TextRange> duplicateTextRanges = new ArrayList<>();
+    Set<TextRange> duplicateTextRanges = new HashSet<>();
     String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
     final EntityType entityType;
 
     @Builder.Default
     final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    final ManualChangeOverwrite manualOverwrite;
+    @Builder.Default
+    final ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
     boolean dictionaryEntry;
     boolean dossierDictionaryEntry;
@@ -68,6 +73,12 @@ public class TextEntity implements IEntity {
     List<SemanticNode> intersectingNodes = new LinkedList<>();
     SemanticNode deepestFullyContainingNode;
 
+    @Builder.Default
+    Map<TextEntity, Set<Relation>> relations = new HashMap<>();
+
+    @Builder.Default
+    Collection<EntityEventListener> entityEventListeners = new ArrayList<>();
+
 
     public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
@@ -158,12 +169,15 @@ public class TextEntity implements IEntity {
 
     public void removeFromGraph() {
 
+        remove("FINAL.0.0", "removed completely");
         intersectingNodes.forEach(node -> node.getEntities().remove(this));
         pages.forEach(page -> page.getEntities().remove(this));
         intersectingNodes = new LinkedList<>();
+        relations.keySet()
+                .forEach(entity -> entity.getRelations().remove(this));
+        relations = new HashedMap<>();
         deepestFullyContainingNode = null;
         pages = new HashSet<>();
-        remove("FINAL.0.0", "removed completely");
     }
 
 
@@ -201,22 +215,20 @@ public class TextEntity implements IEntity {
         return textEntity.contains(this);
     }
 
-
     public boolean contains(TextEntity textEntity) {
 
         if (this.textRange.contains(textEntity.getTextRange())) {
             return true;
         }
 
-        List<TextRange> textEntityDuplicateRanges = textEntity.getDuplicateTextRanges();
-        // use optimized indexed loops for extra performance boost
-        for (int i = 0, duplicateTextRangesSize = duplicateTextRanges.size(); i < duplicateTextRangesSize; i++) {
-            TextRange duplicateTextRange = duplicateTextRanges.get(i);
+        Set<TextRange> textEntityDuplicateRanges = textEntity.getDuplicateTextRanges();
+
+        for (TextRange duplicateTextRange : this.duplicateTextRanges) {
             if (duplicateTextRange.contains(textEntity.getTextRange())) {
                 return true;
             }
-            for (int j = 0, textEntityDuplicateRangesSize = textEntityDuplicateRanges.size(); j < textEntityDuplicateRangesSize; j++) {
-                TextRange otherRange = textEntityDuplicateRanges.get(j);
+
+            for (TextRange otherRange : textEntityDuplicateRanges) {
                 if (duplicateTextRange.contains(otherRange)) {
                     return true;
                 }
@@ -227,6 +239,7 @@ public class TextEntity implements IEntity {
     }
 
 
+
     public boolean intersects(TextEntity textEntity) {
 
         return this.textRange.intersects(textEntity.getTextRange()) //
@@ -251,6 +264,20 @@ public class TextEntity implements IEntity {
     }
 
 
+    public void addManualChange(BaseAnnotation manualChange) {
+
+        manualOverwrite.addChange(manualChange);
+        notifyEntityUpdated();
+    }
+
+
+    public void addManualChanges(List<BaseAnnotation> manualChanges) {
+
+        manualOverwrite.addChanges(manualChanges);
+        notifyEntityUpdated();
+    }
+
+
     public boolean matchesAnnotationId(String manualRedactionId) {
 
         return getPositionsOnPagePerPage().stream()
@@ -311,4 +338,42 @@ public class TextEntity implements IEntity {
                 .orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
     }
 
+
+    @Override
+    public void addEntityEventListener(EntityEventListener listener) {
+
+        entityEventListeners.add(listener);
+    }
+
+
+    @Override
+    public void removeEntityEventListener(EntityEventListener listener) {
+
+        entityEventListeners.remove(listener);
+
+    }
+
+
+    public void computeRelations() {
+
+        for (TextEntity textEntity : this.getDeepestFullyContainingNode().getEntities()) {
+            if (this.intersects(textEntity) && !this.equals(textEntity) && !textEntity.getEntityType().equals(EntityType.TEMPORARY)) {
+                if (textEntity.getTextRange().equals(this.getTextRange())) {
+                    textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Equality(this, textEntity));
+                    this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Equality(textEntity, this));
+                } else if (textEntity.containedBy(this)) {
+                    textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Intersection(textEntity, this));
+                    this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Containment(this, textEntity));
+                } else if (this.containedBy(textEntity)) {
+                    textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Containment(textEntity, this));
+                    this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Intersection(this, textEntity));
+                } else {
+                    textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Intersection(textEntity, this));
+                    this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Intersection(this, textEntity));
+                }
+
+            }
+        }
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java

@@ -9,7 +9,6 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -9,8 +9,8 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveTextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -39,7 +39,6 @@ public class Document extends AbstractSemanticNode {
     @Builder.Default
     static final SectionIdentifier sectionIdentifier = SectionIdentifier.document();
 
-
     @Override
     public NodeType getType() {
 
@@ -169,4 +168,11 @@ public class Document extends AbstractSemanticNode {
         return bBox;
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -1,19 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -67,4 +58,11 @@ public class Footer extends AbstractSemanticNode {
         return getTreeId() + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -1,19 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -70,4 +61,11 @@ public class Header extends AbstractSemanticNode {
         return getTreeId() + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -1,20 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -71,6 +62,11 @@ public class Headline extends AbstractSemanticNode {
     }
 
 
+    /**
+     * Extracts the SectionIdentifier from the text of this headline.
+     *
+     * @return The SectionIdentifier, with which the headline starts.
+     */
     @Override
     public SectionIdentifier getSectionIdentifier() {
 
@@ -104,4 +100,11 @@ public class Headline extends AbstractSemanticNode {
                 .isPresent();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -1,24 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityEventListener;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -30,8 +28,7 @@ import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;
 
 /**
- *
- Represents an image within the document.
+ * Represents an image within the document.
  */
 @Data
 @SuperBuilder
@@ -43,6 +40,7 @@ public class Image extends AbstractSemanticNode implements IEntity {
 
     String id;
 
+    String representationHash;
     TextBlock leafTextBlock;
 
     ImageType imageType;
@@ -57,6 +55,9 @@ public class Image extends AbstractSemanticNode implements IEntity {
 
     Page page;
 
+    @Builder.Default
+    Collection<EntityEventListener> entityEventListeners = new ArrayList<>();
+
 
     @Override
     public NodeType getType() {
@@ -79,6 +80,18 @@ public class Image extends AbstractSemanticNode implements IEntity {
     }
 
 
+    public boolean isFullPageImage() {
+
+        return imageType.equals(ImageType.OCR) || getArea() >= 0.5 * page.getArea();
+    }
+
+
+    private double getArea() {
+
+        return position.getWidth() * position.getHeight();
+    }
+
+
     @Override
     public TextRange getTextRange() {
 
@@ -93,17 +106,33 @@ public class Image extends AbstractSemanticNode implements IEntity {
     }
 
 
+    @Override
+    public void addEntityEventListener(EntityEventListener listener) {
+
+        entityEventListeners.add(listener);
+    }
+
+
+    @Override
+    public void removeEntityEventListener(EntityEventListener listener) {
+
+        entityEventListeners.remove(listener);
+
+    }
+
+
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(imageType.toString().toLowerCase(Locale.ENGLISH));
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString().toLowerCase(Locale.ENGLISH));
     }
 
 
     @Override
     public String toString() {
 
-        return getTreeId() + ": " + getValue() + " " + position;
+        return getTreeId() + ": " + getValue() + " [%.2f,%.2f,%.2f,%.2f]".formatted(position.getX(), position.getY(), position.getWidth(), position.getHeight());
     }
 
 
@@ -154,4 +183,18 @@ public class Image extends AbstractSemanticNode implements IEntity {
         return (area / calculatedIntersection) > containmentThreshold;
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
+
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java

@@ -0,0 +1,7 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+public enum LayoutEngine {
+    ALGORITHM,
+    AI,
+    OUTLINE
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java

@@ -12,7 +12,9 @@ public enum NodeType {
     TABLE_CELL,
     IMAGE,
     HEADER,
-    FOOTER;
+    FOOTER,
+    TABLE_OF_CONTENTS,
+    TABLE_OF_CONTENTS_ITEM;
 
 
     public String toString() {

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java

@@ -118,4 +118,10 @@ public class Page {
         return "Page: " + number;
     }
 
+
+    public double getArea() {
+
+        return width * height;
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -1,19 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -59,4 +50,11 @@ public class Paragraph extends AbstractSemanticNode {
         return getTreeId() + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -1,23 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;
 import lombok.extern.slf4j.Slf4j;
@@ -33,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
 public class Section extends AbstractSemanticNode {
 
-
     @Override
     public NodeType getType() {
 
@@ -53,6 +40,11 @@ public class Section extends AbstractSemanticNode {
     }
 
 
+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
     @Override
     public SectionIdentifier getSectionIdentifier() {
 
@@ -60,7 +52,6 @@ public class Section extends AbstractSemanticNode {
     }
 
 
-
     @Override
     public String toString() {
 
@@ -101,4 +92,10 @@ public class Section extends AbstractSemanticNode {
     }
 
 
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java

@@ -13,13 +13,14 @@ import lombok.Getter;
 import lombok.experimental.FieldDefaults;
 
 /**
- * Represents a unique identifier for a section within a document.
+ * Represents the textual identifier sometimes present in a Headline. For example, given the headline 3.1 Results, the section identifier is 3.1.
+ * Keep in mind, this identifier must not be unique in a single document, as there might be multiple headlines starting with the same textual identifier.
  */
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
 
-    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?");
     public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");
 
     protected enum Format {
@@ -142,7 +143,7 @@ public class SectionIdentifier {
 
 
     /**
-     * Determines if the current section is the parent of the given section.
+     * Determines if the current SectionIdentifier is the parent of the given SectionIdentifier.
      *
      * @param sectionIdentifier The section identifier to compare against.
      * @return true if the current section is the parent of the given section, false otherwise.

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -14,18 +14,17 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveTextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConsecutiveTextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -182,7 +181,7 @@ public interface SemanticNode {
 
 
     /**
-     * Returns a SectionIdentifier, such that it acts as a child of the first Headline associated with this SemanticNode.
+     * Returns the SectionIdentifier as a child of the SectionIdentifier returned by the getHeadline() method.
      *
      * @return The SectionIdentifier from the first Headline.
      */
@@ -819,16 +818,12 @@ public interface SemanticNode {
 
     /**
      * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
-     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
+     * The visitor's {@link NodeVisitor#visit} method is invoked for each node encountered during the traversal.
      *
      * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
      * @see NodeVisitor
      */
-    default void accept(NodeVisitor visitor) {
-
-        visitor.visit(this);
-        streamChildren().forEach(childNode -> childNode.accept(visitor));
-    }
+    void accept(NodeVisitor visitor);
 
 
     /**

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java

@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
+
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -39,6 +40,11 @@ public class SuperSection extends AbstractSemanticNode {
     }
 
 
+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
     @Override
     public SectionIdentifier getSectionIdentifier() {
 
@@ -46,7 +52,6 @@ public class SuperSection extends AbstractSemanticNode {
     }
 
 
-
     @Override
     public String toString() {
 
@@ -87,4 +92,10 @@ public class SuperSection extends AbstractSemanticNode {
     }
 
 
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -14,10 +14,10 @@ import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -419,4 +419,11 @@ public class Table implements SemanticNode {
         return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -4,6 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
 
@@ -80,4 +81,11 @@ public class TableCell extends AbstractSemanticNode {
         return getTreeId() + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
     }
 
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
 }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableOfContents.java

@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class TableOfContents extends AbstractSemanticNode {
+
+    @Override
+    public NodeType getType() {
+
+        return NodeType.TABLE_OF_CONTENTS;
+    }
+
+
+    public Headline getHeadline() {
+
+        return streamChildrenOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElseGet(() -> getParent().getHeadline());
+    }
+
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
+
+    @Override
+    public String toString() {
+
+        return getTreeId() + ": " + NodeType.TABLE_OF_CONTENTS_ITEM + ": " + getTextBlock().buildSummary();
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableOfContentsItem.java

@@ -0,0 +1,57 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class TableOfContentsItem extends AbstractSemanticNode {
+
+    TextBlock leafTextBlock;
+
+
+    @Override
+    public NodeType getType() {
+
+        return NodeType.TABLE_OF_CONTENTS_ITEM;
+    }
+
+
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
+    @Override
+    public void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        return leafTextBlock;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return getTreeId() + ": " + NodeType.TABLE_OF_CONTENTS_ITEM + ": " + leafTextBlock.buildSummary();
+    }
+
+}

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 
+import static com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.DocumentPositionData;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
 import java.lang.ref.SoftReference;
 import java.text.BreakIterator;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -16,12 +16,12 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto.DocumentPositionData.Position;
+import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto.DocumentTextData;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -29,6 +29,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.experimental.FieldDefaults;
 
 @Data
@@ -37,21 +38,33 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class AtomicTextBlock implements TextBlock {
 
+    @NonNull
     Long id;
+    @NonNull
     Integer numberOnPage;
+    @NonNull
     Page page;
 
     //string coordinates
+    @NonNull
     TextRange textRange;
+    @NonNull
     String searchText;
+    @NonNull
     List<Integer> lineBreaks;
+    @NonNull
+    List<TextRange> italicTextRanges;
+    @NonNull
+    List<TextRange> boldTextRanges;
 
     SoftReference<String> searchTextLowerCaseCache;
     SoftReference<List<String>> wordsCache;
 
     //position coordinates
+    @NonNull
     List<Integer> stringIdxToPositionIdx;
     @Getter
+    @NonNull
     List<Rectangle2D> positions;
 
     @EqualsAndHashCode.Exclude
@@ -77,6 +90,8 @@ public class AtomicTextBlock implements TextBlock {
                 .stringIdxToPositionIdx(Collections.emptyList())
                 .positions(Collections.emptyList())
                 .parent(parent)
+                .boldTextRanges(Collections.emptyList())
+                .italicTextRanges(Collections.emptyList())
                 .build();
     }
 
@@ -89,20 +104,26 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                 .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed()
-                                    .toList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed()
-                                                .toList())
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
+                .lineBreaks(atomicTextBlockData.getLineBreaksList())
+                .stringIdxToPositionIdx(atomicPositionBlockData.getStringIdxToPositionIdxList())
+                .positions(toRectangle2DList(atomicPositionBlockData.getPositionsList()))
+                .italicTextRanges(atomicTextBlockData.getItalicTextRangesList()
+                                          .stream()
+                                          .map(r -> new TextRange(r.getStart(), r.getEnd()))
+                                          .toList())
+                .boldTextRanges(atomicTextBlockData.getBoldTextRangesList()
+                                        .stream()
+                                        .map(r -> new TextRange(r.getStart(), r.getEnd()))
+                                        .toList())
                 .parent(parent)
                 .build();
     }
 
 
-    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
+    private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {
 
-        return Arrays.stream(positions)
-                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+        return positions.stream()
+                .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getValue(0), pos.getValue(1), pos.getValue(2), pos.getValue(3)))
                 .toList();
     }
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.Data;
+import lombok.NonNull;
 import lombok.experimental.FieldDefaults;
 
 @Data
@@ -162,6 +163,26 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+    @Override
+    public List<TextRange> getItalicTextRanges() {
+
+        return getAtomicTextBlocks().stream()
+                .flatMap(atomicTextBlock -> atomicTextBlock.getItalicTextRanges()
+                        .stream())
+                .toList();
+    }
+
+
+    @Override
+    public List<TextRange> getBoldTextRanges() {
+
+        return getAtomicTextBlocks().stream()
+                .flatMap(atomicTextBlock -> atomicTextBlock.getBoldTextRanges()
+                        .stream())
+                .toList();
+    }
+
+
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
@@ -279,6 +300,7 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+    @NonNull
     @Override
     public String toString() {
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConsecutiveTextBlockCollector.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document;
+package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -10,9 +10,6 @@ import java.util.function.Supplier;
 import java.util.stream.Collector;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-
 import lombok.NoArgsConstructor;
 
 @NoArgsConstructor

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -55,6 +55,12 @@ public interface TextBlock extends CharSequence {
     String subSequenceWithLineBreaks(TextRange textRange);
 
 
+    List<TextRange> getItalicTextRanges();
+
+
+    List<TextRange> getBoldTextRanges();
+
+
     int numberOfLines();
 
 

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java

@@ -1,23 +1,27 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.utils;
 
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.server.model.document.IntersectingNodeVisitor;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class EntityCreationUtility {
 
-    public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
+    public void checkIfBothStartAndEndAreEmpty(String start, String end) {
 
         checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
     }
 
 
-    public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+    public <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
 
         if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
             throw new IllegalArgumentException("Start and end values are empty!");
@@ -25,7 +29,7 @@ public class EntityCreationUtility {
     }
 
 
-    public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+    public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
 
         if (textBlock.getNextLinebreak(end) < expandedEnd) {
             return end;
@@ -34,7 +38,7 @@ public class EntityCreationUtility {
     }
 
 
-    public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
+    public Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
 
         IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
 
@@ -46,7 +50,7 @@ public class EntityCreationUtility {
     }
 
 
-    public static void addToPages(TextEntity entity) {
+    public void addToPages(TextEntity entity) {
 
         Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
         entity.getPages().addAll(pages);
@@ -54,14 +58,14 @@ public class EntityCreationUtility {
     }
 
 
-    public static void addEntityToNodeEntitySets(TextEntity entity) {
+    public void addEntityToNodeEntitySets(TextEntity entity) {
 
         entity.getIntersectingNodes()
                 .forEach(node -> node.getEntities().add(entity));
     }
 
 
-    public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
+    public boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
 
         if (entitiesToMerge.isEmpty()) {
             return true;
@@ -79,7 +83,7 @@ public class EntityCreationUtility {
     }
 
 
-    public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
+    public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
 
         if (textBlock.getTextRange().end() == textRange.end()) {
             return new TextRange(textRange.end(), textRange.end());

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java

@@ -1,39 +1,36 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.utils;
 
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 
-import lombok.RequiredArgsConstructor;
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.UtilityClass;
 
-@Service
-@RequiredArgsConstructor
+@UtilityClass
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityEnrichmentService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
+    int SURROUNDING_WORDS_OFFSET_WINDOW = 100;
+    int NUMBER_OF_SURROUNDING_WORDS = 3;
 
 
     public void enrichEntity(TextEntity entity, TextBlock textBlock) {
-
         entity.setValue(textBlock.subSequence(entity.getTextRange()).toString());
         entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock));
         entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock));
     }
 
-
     private String findTextAfter(int index, TextBlock textBlock) {
-
-        int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end());
+        int endOffset = Math.min(index + SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().end());
         String textAfter = textBlock.subSequence(index, endOffset).toString();
         if (!textAfter.isBlank()) {
             List<String> wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter);
-            int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords());
+            int numberOfWordsAfter = Math.min(wordsAfter.size(), NUMBER_OF_SURROUNDING_WORDS);
             if (!wordsAfter.isEmpty()) {
                 return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" "));
             }
@@ -41,14 +38,12 @@ public class EntityEnrichmentService {
         return "";
     }
 
-
     private String findTextBefore(int index, TextBlock textBlock) {
-
-        int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start());
+        int offsetBefore = Math.max(index - SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().start());
         String textBefore = textBlock.subSequence(offsetBefore, index).toString();
         if (!textBefore.isBlank()) {
             List<String> wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore);
-            int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords());
+            int numberOfWordsBefore = Math.min(wordsBefore.size(), NUMBER_OF_SURROUNDING_WORDS);
             if (!wordsBefore.isEmpty()) {
                 return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" "));
             }
@@ -56,36 +51,26 @@ public class EntityEnrichmentService {
         return "";
     }
 
-
-    private static List<String> splitToWordsAndRemoveEmptyWords(String textAfter) {
-
-        return Arrays.stream(textAfter.split(" "))
+    private List<String> splitToWordsAndRemoveEmptyWords(String text) {
+        return Arrays.stream(text.split(" "))
                 .filter(word -> !Objects.equals("", word))
                 .toList();
     }
 
-
-    private static String concatWordsBefore(List<String> words, boolean endWithSpace) {
-
+    private String concatWordsBefore(List<String> words, boolean endWithSpace) {
         StringBuilder sb = new StringBuilder();
-
         for (String word : words) {
             sb.append(word).append(" ");
         }
-
         String result = sb.toString().trim();
         return endWithSpace ? result + " " : result;
     }
 
-
-    private static String concatWordsAfter(List<String> words, boolean startWithSpace) {
-
+    private String concatWordsAfter(List<String> words, boolean startWithSpace) {
         StringBuilder sb = new StringBuilder();
-
         for (String word : words) {
             sb.append(word).append(" ");
         }
-
         String result = sb.toString().trim();
         return startWithSpace ? " " + result : result;
     }

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java

@@ -313,7 +313,7 @@ public class RedactionSearchUtility {
             while (matcher.find()) {
                 boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
             }
-        }catch (StackOverflowError stackOverflowError){
+        } catch (StackOverflowError stackOverflowError) {
             log.warn("Stackoverflow error for pattern {} in text: {}", pattern.pattern(), textBlock);
         }
         return boundaries;
@@ -322,7 +322,6 @@ public class RedactionSearchUtility {
 
     private static List<TextRange> getTextRangesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) {
 
-
         String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks();
         Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
         List<TextRange> boundaries = new LinkedList<>();
@@ -330,7 +329,7 @@ public class RedactionSearchUtility {
             while (matcher.find()) {
                 boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
             }
-        }catch (StackOverflowError stackOverflowError){
+        } catch (StackOverflowError stackOverflowError) {
             log.warn("Stackoverflow error for pattern {} in text with linebreaks: {}", pattern.pattern(), searchTextWithLineBreaks);
         }
         return boundaries;

redaction-service-v1/document/src/main/resources/DocumentPage.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+option java_outer_classname = "DocumentPageProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+
+message AllDocumentPages {
+
+  repeated DocumentPage documentPages = 1;
+}
+
+message DocumentPage {
+  // The page number, starting with 1.
+  int32 number = 1;
+
+  // The page height in PDF user units.
+  int32 height = 2;
+
+  // The page width in PDF user units.
+  int32 width = 3;
+
+  // The page rotation as specified by the PDF.
+  int32 rotation = 4;
+}
+

redaction-service-v1/document/src/main/resources/DocumentPositionData.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+option java_outer_classname = "DocumentPositionDataProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+message AllDocumentPositionData {
+
+  repeated DocumentPositionData documentPositionData = 1;
+}
+
+message DocumentPositionData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate.
+  // This is required due to the text and position coordinates not being equal.
+  repeated int32 stringIdxToPositionIdx = 2;
+
+  // The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block.
+  // The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner.
+  // In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.
+  repeated Position positions = 3;
+
+  // Definition of a BoundingBox that contains x, y, width, and height.
+  message Position {
+    repeated float value = 1;
+  }
+}

redaction-service-v1/document/src/main/resources/DocumentStructure.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+
+option java_outer_classname = "DocumentStructureProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+
+import "EntryData.proto";
+
+message DocumentStructure {
+  // The root EntryData represents the Document.
+  EntryData root = 1;
+}

redaction-service-v1/document/src/main/resources/DocumentTextData.proto

@@ -0,0 +1,40 @@
+syntax = "proto3";
+
+import "Range.proto";
+
+option java_outer_classname = "DocumentTextDataProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+message AllDocumentTextData {
+
+  repeated DocumentTextData documentTextData = 1;
+}
+
+message DocumentTextData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // The page the text block occurs on.
+  int64 page = 2;
+
+  // The text of the text block.
+  string searchText = 3;
+
+  // Each text block is assigned a number on a page, starting from 0.
+  int32 numberOnPage = 4;
+
+  // The text blocks are ordered, this number represents the start of the text block as a string offset.
+  int32 start = 5;
+
+  // The text blocks are ordered, this number represents the end of the text block as a string offset.
+  int32 end = 6;
+
+  // The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.
+  repeated int32 lineBreaks = 7;
+
+  // The text ranges where the text is italic
+  repeated Range italicTextRanges = 8;
+
+  // The text ranges where the text is bold
+  repeated Range boldTextRanges = 9;
+}

redaction-service-v1/document/src/main/resources/EntryData.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+import "LayoutEngine.proto";
+import "NodeType.proto";
+
+option java_outer_classname = "EntryDataProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+message EntryData {
+  // Type of the semantic node.
+  NodeType type = 1;
+
+  // Specifies the position in the parsed tree structure.
+  repeated int32 treeId = 2;
+
+  // Specifies the text block IDs associated with this semantic node.
+  repeated int64 atomicBlockIds = 3;
+
+  // Specifies the pages this semantic node appears on.
+  repeated int64 pageNumbers = 4;
+
+  // Some semantic nodes have additional information, this information is stored in this Map.
+  map<string, string> properties = 5;
+
+  // All child Entries of this Entry.
+  repeated EntryData children = 6;
+
+  // Describes the origin of the semantic node.
+  repeated LayoutEngine engines = 7;
+}

redaction-service-v1/document/src/main/resources/LayoutEngine.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+option java_outer_classname = "LayoutEngineProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+enum LayoutEngine {
+  ALGORITHM = 0;
+  AI = 1;
+  OUTLINE = 2;
+}

redaction-service-v1/document/src/main/resources/NodeType.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+option java_outer_classname = "NodeTypeProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+enum NodeType {
+  DOCUMENT = 0;
+  SECTION = 1;
+  SUPER_SECTION = 2;
+  HEADLINE = 3;
+  PARAGRAPH = 4;
+  TABLE = 5;
+  TABLE_CELL = 6;
+  IMAGE = 7;
+  HEADER = 8;
+  FOOTER = 9;
+  TABLE_OF_CONTENTS = 10;
+  TABLE_OF_CONTENTS_ITEM = 11;
+}

redaction-service-v1/document/src/main/resources/Range.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+option java_outer_classname = "RangeProto";
+option java_package = "com.iqser.red.service.redaction.v1.server.data";
+
+
+
+message Range {
+  // A start index.
+  int32 start = 1;
+
+  // An end index.
+  int32 end = 2;
+}

redaction-service-v1/document/src/main/resources/generate_proto_files.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Minimum required protoc version
+MIN_VERSION="28.3"
+
+# Get the installed protoc version
+INSTALLED_VERSION=$(protoc --version | awk '{print $2}')
+
+# Function to compare versions
+version_lt() {
+    [ "$(printf '%s\n' "$1" "$2" | sort -V | head -n1)" != "$1" ]
+}
+
+# Check if protoc is installed and meets the minimum version
+if ! command -v protoc &> /dev/null; then
+    echo "Error: protoc is not installed. Please install version $MIN_VERSION or later."
+    exit 1
+fi
+
+if version_lt "$INSTALLED_VERSION" "$MIN_VERSION"; then
+    echo "Error: protoc version $INSTALLED_VERSION is too old. Please upgrade to version $MIN_VERSION or later."
+    exit 1
+fi
+
+# Generate Java files from proto files
+protoc --java_out=../java ./*.proto

redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto;
+
+public class LayoutEngineMappingTest {
+
+    @Test
+    public void assertAllValuesMatch() {
+
+        for (LayoutEngine value : LayoutEngine.values()) {
+            var engine = LayoutEngineProto.LayoutEngine.valueOf(value.name());
+            assertEquals(engine.name(), value.name());
+        }
+    }
+
+
+    @Test
+    public void assertAllValuesMatchReverse() {
+
+        for (LayoutEngineProto.LayoutEngine value : LayoutEngineProto.LayoutEngine.values()) {
+            if (value.equals(LayoutEngineProto.LayoutEngine.UNRECOGNIZED)) {
+                continue;
+            }
+            var engine = LayoutEngine.valueOf(value.name());
+            assertEquals(engine.name(), value.name());
+        }
+    }
+
+}

redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto;
+
+public class NodeTypeMappingTest {
+
+    @Test
+    public void assertAllValuesMatch() {
+
+        for (NodeType value : NodeType.values()) {
+            var engine = NodeTypeProto.NodeType.valueOf(value.name());
+            assertEquals(engine.name(), value.name());
+        }
+    }
+
+
+    @Test
+    public void assertAllValuesMatchReverse() {
+
+        for (NodeTypeProto.NodeType value : NodeTypeProto.NodeType.values()) {
+            if (value.equals(NodeTypeProto.NodeType.UNRECOGNIZED)) {
+                continue;
+            }
+            var engine = NodeType.valueOf(value.name());
+            assertEquals(engine.name(), value.name());
+        }
+    }
+
+}

redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifierTest.java

@@ -0,0 +1,144 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+class SectionIdentifierTest {
+
+    @Test
+    void testSectionIdentifier() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("1.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
+
+        SectionIdentifier child = SectionIdentifier.asChildOf(identifier);
+        assertTrue(child.isChildOf(identifier));
+
+        SectionIdentifier parent = SectionIdentifier.fromSearchText("1.1: Headline");
+        assertTrue(parent.isParentOf(identifier));
+    }
+
+
+    @Test
+    void testSectionIdentifier2() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("A.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier3() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(4, 1, 2), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier4() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier5() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier6() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("d.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier7() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testFalsePositive111() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("111: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(1, identifier.level());
+    }
+
+
+    @Test
+    public void testParentOf() {
+
+        var headline = SectionIdentifier.fromSearchText("1 Did you ever hear the tragedy of Darth Plagueis The Wise?");
+        var headline1 = SectionIdentifier.fromSearchText("1.0 I thought not. It’s not a story the Jedi would tell you.");
+        var headline2 = SectionIdentifier.fromSearchText("1.1  It’s a Sith legend. Darth Plagueis was a Dark Lord of the Sith, ");
+        var headline3 = SectionIdentifier.fromSearchText("1.2.3 so powerful and so wise he could use the Force to influence the midichlorians to create life…");
+        var headline4 = SectionIdentifier.fromSearchText("1.2.3.4 He had such a knowledge of the dark side that he could even keep the ones he cared about from dying.");
+        var headline5 = SectionIdentifier.fromSearchText("1.2.3.4.5 The dark side of the Force is a pathway to many abilities some consider to be unnatural.");
+        var headline6 = SectionIdentifier.fromSearchText("2.0 He became so powerful…");
+        var headline7 = SectionIdentifier.fromSearchText("10000.0 the only thing he was afraid of was losing his power,");
+        var headline8 = SectionIdentifier.fromSearchText("A.0 which eventually, of course, he did.");
+        var headline9 = SectionIdentifier.fromSearchText("Unfortunately, he taught his apprentice everything he knew, then his apprentice killed him in his sleep.");
+        var headline10 = SectionIdentifier.fromSearchText("2.1.2 Ironic.");
+        var headline11 = SectionIdentifier.fromSearchText("2.He could save others from death,");
+        var headline12 = SectionIdentifier.fromSearchText(" 2. but not himself.");
+
+        var paragraph1 = SectionIdentifier.asChildOf(headline);
+        assertTrue(paragraph1.isChildOf(headline));
+        assertTrue(headline.isParentOf(paragraph1));
+        assertFalse(paragraph1.isParentOf(headline));
+
+        assertFalse(headline.isParentOf(headline1));
+        assertTrue(headline.isParentOf(headline2));
+        assertTrue(headline.isParentOf(headline3));
+        assertTrue(headline.isParentOf(headline4));
+        assertTrue(headline.isParentOf(headline5));
+        assertTrue(headline1.isParentOf(headline2));
+        assertFalse(headline1.isParentOf(headline1));
+        assertTrue(headline3.isParentOf(headline4));
+        assertFalse(headline4.isParentOf(headline5));
+        assertFalse(headline2.isParentOf(headline3));
+        assertFalse(headline2.isParentOf(headline4));
+        assertTrue(headline1.isParentOf(headline3));
+        assertTrue(headline1.isParentOf(headline4));
+        assertFalse(headline1.isParentOf(headline6));
+        assertFalse(headline1.isParentOf(headline7));
+        assertFalse(headline8.isParentOf(headline1));
+        assertFalse(headline8.isParentOf(headline2));
+        assertFalse(headline8.isParentOf(headline3));
+        assertFalse(headline8.isParentOf(headline4));
+        assertFalse(headline9.isParentOf(headline9));
+        assertTrue(headline10.isChildOf(headline11));
+        assertTrue(headline10.isChildOf(headline12));
+    }
+
+}

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,11 +4,13 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.580.0"
+val persistenceServiceVersion = rootProject.extra.get("persistenceServiceVersion")
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") {
+        exclude(group = "com.knecon.fforesight", module = "document")
+    }
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java

@@ -1,13 +1,26 @@
 package com.iqser.red.service.redaction.v1.resources;
 
-import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
-
 import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest;
+import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
 
 public interface RuleBuilderResource {
 
     @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE)
     RuleBuilderModel getRuleBuilderModel();
 
+
+    @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
+    RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest);
+
+
+    @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
+    ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest);
+
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,12 +12,12 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.174.0"
+val layoutParserVersion = "0.193.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.580.0"
-val llmServiceVersion = "1.11.0"
+val persistenceServiceVersion = rootProject.extra.get("persistenceServiceVersion")
+val llmServiceVersion = "1.20.0-RED10072.2"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
@@ -34,10 +34,14 @@ configurations {
 dependencies {
 
     implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
-    implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}")
-    {
+    implementation(project(":document"))
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") {
+        exclude(group = "org.springframework.boot")
+        exclude(group = "com.knecon.fforesight", module = "document")
+    }
+    implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") {
         exclude(group = "com.knecon.fforesight", module = "tenant-commons")
+        exclude(group = "com.knecon.fforesight", module = "document")
     }
     implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
     implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}")
@@ -45,8 +49,8 @@ dependencies {
     implementation("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.30.0")
+    implementation("com.iqser.red.commons:storage-commons:2.50.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.31.0")
     implementation("com.knecon.fforesight:keycloak-commons:0.30.0") {
         exclude(group = "com.knecon.fforesight", module = "tenant-commons")
     }
@@ -55,7 +59,9 @@ dependencies {
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.ahocorasick:ahocorasick:0.6.3")
+    implementation("org.ahocorasick:ahocorasick:0.9.0")
+    implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2")
+    implementation("com.github.roklenarcic:aho-corasick:1.2")
     implementation("org.javassist:javassist:3.29.2-GA")
 
     implementation("org.drools:drools-engine:${droolsVersion}")
@@ -74,6 +80,9 @@ dependencies {
     implementation("org.apache.tomcat:tomcat-websocket:${tomcatVersion}")
     implementation("org.apache.tomcat.embed:tomcat-embed-core:${tomcatVersion}")
 
+    implementation("org.liquibase:liquibase-core:4.29.2") // Needed to be set explicit, otherwise spring dependency management sets it to 4.20.0
+    implementation("org.liquibase.ext:liquibase-mongodb:4.29.2")
+
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     api("ch.qos.logback:logback-classic")
 
@@ -92,10 +101,8 @@ dependencies {
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
     testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
-        exclude(
-            group = "com.iqser.red.service",
-            module = "persistence-service-shared-api-v1"
-        )
+        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
+        exclude(group = "com.knecon.fforesight", module = "document")
     }
     testImplementation("com.pdftron:PDFNet:10.11.0")
 }
@@ -119,9 +126,9 @@ tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put(
-        "BPE_APPEND_JAVA_TOOL_OPTIONS",
-        "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5"
+        "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5"
     )
+    environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding
 
     imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
     if (project.hasProperty("buildbootDockerHostNetwork")) {
@@ -145,44 +152,50 @@ tasks.named<BootBuildImage>("bootBuildImage") {
     }
 }
 
-fun parseDroolsImports(vararg droolsFilePaths: String): List<String> {
-    val imports = mutableListOf<String>()
-    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
-    val desiredPrefix = "com.iqser.red.service.redaction.v1"
-
-    droolsFilePaths.forEach { filePath ->
-        File(filePath).forEachLine { line ->
-            importPattern.find(line)?.let { matchResult ->
-                val importPath = matchResult.groupValues[1].trim()
-                if (importPath.startsWith(desiredPrefix)) {
-                    val formattedPath = importPath.replace('.', '/')
-                    imports.add("$formattedPath.java")
-                }
-            }
-        }
-    }
-
-    return imports
-}
+//fun parseDroolsImports(vararg droolsFilePaths: String): List<String> {
+//    val imports = mutableListOf<String>()
+//    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+//    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+//
+//    droolsFilePaths.forEach { filePath ->
+//        File(filePath).forEachLine { line ->
+//            importPattern.find(line)?.let { matchResult ->
+//                val importPath = matchResult.groupValues[1].trim()
+//                if (importPath.startsWith(desiredPrefix)) {
+//                    val formattedPath = importPath.replace('.', '/')
+//                    imports.add("$formattedPath.java")
+//                }
+//            }
+//        }
+//    }
+//
+//    return imports
+//}
 
 // Combine imports from both drools files
-val droolsImports = parseDroolsImports(
-    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl",
-    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl"
-)
+//val droolsImports = parseDroolsImports(
+//    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl",
+//    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl"
+//)
 
 tasks.register("generateJavaDoc", Javadoc::class) {
 
     dependsOn("compileJava")
     dependsOn("delombok")
     classpath = project.sourceSets["main"].runtimeClasspath
-    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
-        include(droolsImports)
-    }
-    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+//    val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") {
+//        include(droolsImports)
+//    }
+//    val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") {
+//        include(droolsImports)
+//    }
+//    source = documentFiles + mainFiles
+
+    setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "javadoc"))
 
     options.memberLevel = JavadocMemberLevel.PUBLIC
     (options as StandardJavadocDocletOptions).apply {
         title = "API Documentation for Redaction Service ${project.version}"
     }
 }
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -10,11 +10,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.reflections.Reflections;
 import org.reflections.scanners.Scanners;
 import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
 
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 
@@ -25,6 +27,8 @@ import lombok.extern.slf4j.Slf4j;
 public class DeprecatedElementsFinder {
 
     public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    public static final Pattern DATA_PACKAGE = Pattern.compile(".*/data/.*");
+
     private Set<Method> deprecatedMethods;
     @Getter
     private Map<String, String> deprecatedMethodsSignaturesMap;
@@ -43,7 +47,10 @@ public class DeprecatedElementsFinder {
 
         Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
                                                           .setExpandSuperTypes(true)
-                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes)
+                                                          .filterInputsBy(new FilterBuilder().includePackage(PACKAGE_NAME).excludePackage(PACKAGE_NAME + ".data")
+                                                                          // Exclude the generated proto data package
+                                                          ));
 
         deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -28,11 +28,13 @@ public class RedactionServiceSettings {
 
     private boolean priorityMode;
 
+    private long firstLevelDictionaryCacheMaximumSize = 1000;
+
     private long dictionaryCacheMaximumSize = 100;
 
     private int dictionaryCacheExpireAfterAccessDays = 3;
 
-    private int droolsExecutionTimeoutSecs = 300;
+    private int droolsExecutionTimeoutSecs = 600;
 
     private boolean ruleExecutionSecured = true;
 
@@ -40,6 +42,8 @@ public class RedactionServiceSettings {
 
     private boolean droolsDebug;
 
+    private boolean protobufJsonFallback = true;
+
 
     public int getDroolsExecutionTimeoutSecs(int numberOfPages) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java

@@ -1,12 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.configuraton;
-
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
-
-@Configuration
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
-public class PriorityTenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java

@@ -1,12 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.configuraton;
-
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
-
-@Configuration
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
-public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java

@@ -1,27 +1,113 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.util.Collections;
-
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest;
 import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
+import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource;
+import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult;
+import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService;
+import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @RestController
+@Slf4j
 @RequiredArgsConstructor
 public class RuleBuilderController implements RuleBuilderResource {
 
+    private final RuleBuilderService ruleBuilderService;
+    private final DroolsValidationService droolsValidationService;
+
+
     @Override
     public RuleBuilderModel getRuleBuilderModel() {
 
-        RuleBuilderModel ruleBuilderModel = new RuleBuilderModel();
+        return this.ruleBuilderService.getRuleBuilderModel();
+    }
 
-        ruleBuilderModel.setWhenClauses(Collections.emptyList());
-        ruleBuilderModel.setThenConditions(Collections.emptyList());
 
-        return ruleBuilderModel;
+    @Override
+    public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) {
+
+        RulesResponse rulesResponse = new RulesResponse();
+        String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true);
+        if (filteredRules.isEmpty()) {
+            throw new AssertionError("There was an error when cleaning the rulefile of sytem rules");
+        }
+        rulesResponse.setRules(filteredRules);
+        return rulesResponse;
+    }
+
+
+    @Override
+    public ResponseEntity<RulesUploadResponse> mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) {
+
+        RulesUploadResponse rulesUploadResponse = new RulesUploadResponse();
+        DroolsValidationResponse droolsValidationResponse;
+        RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules());
+        if (mergingResult.getMergedRules().isEmpty()) {
+            throw new AssertionError("There was an error when merging the user rule update into the rule file");
+        }
+        try {
+            var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules()));
+            droolsValidationResponse = DroolsValidationResponse.builder()
+                    .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages()
+                                                 .stream()
+                                                 .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine()
+                                                                                                             - (mergingResult.getAddedImportsOffset()
+                                                                                                                + mergingResult.getAddedGlobalsOffset()),
+                                                                                                             droolsSyntaxErrorMessage.getColumn(),
+                                                                                                             droolsSyntaxErrorMessage.getMessage()))
+                                                 .toList())
+                    .deprecatedWarnings(droolsValidation.getDeprecatedWarnings()
+                                                .stream()
+                                                .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine()
+                                                                                                                    - (mergingResult.getAddedImportsOffset()
+                                                                                                                       + mergingResult.getAddedGlobalsOffset()),
+                                                                                                                    droolsSyntaxDeprecatedWarnings.getColumn(),
+                                                                                                                    droolsSyntaxDeprecatedWarnings.getMessage()))
+                                                .toList())
+                    .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages()
+                                                    .stream()
+                                                    .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine()
+                                                                                                                      - (mergingResult.getAddedImportsOffset()
+                                                                                                                         + mergingResult.getAddedGlobalsOffset()),
+                                                                                                                      droolsBlacklistErrorMessage.getColumn(),
+                                                                                                                      droolsBlacklistErrorMessage.getMessage()))
+                                                    .toList())
+                    .build();
+            if (!droolsValidation.isCompiled()) {
+                rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse);
+                return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY);
+            } else {
+                rulesUploadResponse.setRules(mergingResult.getMergedRules());
+                rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse);
+            }
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
+        }
+        log.info("response body: " + rulesUploadResponse);
+        log.info("response: " + new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK));
+        return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK);
+    }
+
+
+    private enum RuleFileType {
+        ENTITY,
+        COMPONENT
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java

@@ -82,4 +82,14 @@ public class NerEntities {
         LLM_NER
     }
 
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) {
+
+        return switch (nerEntityEngine) {
+            case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER;
+        };
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model;
 
 import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.PriorityQueue;
@@ -15,10 +16,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityEventListener;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.RectangleWithPage;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -51,7 +54,8 @@ public class PrecursorEntity implements IEntity {
 
     @Builder.Default
     PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    ManualChangeOverwrite manualOverwrite;
+    @Builder.Default
+    ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
 
     public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
@@ -125,6 +129,7 @@ public class PrecursorEntity implements IEntity {
                 .id(importedRedaction.getId())
                 .value(value)
                 .entityPosition(rectangleWithPages)
+                .ruleIdentifier("IMP.0.0")
                 .reason(Optional.ofNullable(importedRedaction.getReason())
                                 .orElse(""))
                 .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
@@ -196,6 +201,28 @@ public class PrecursorEntity implements IEntity {
     }
 
 
+    @Override
+    public void addEntityEventListener(EntityEventListener listener) {
+
+        throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners");
+    }
+
+
+    @Override
+    public void removeEntityEventListener(EntityEventListener listener) {
+
+        throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners");
+
+    }
+
+
+    @Override
+    public Collection<EntityEventListener> getEntityEventListeners() {
+
+        throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners");
+    }
+
+
     private static EntityType getEntityType(EntryType entryType) {
 
         switch (entryType) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class RuleMergingResult {
+
+    private String mergedRules;
+
+    private int addedImportsOffset;
+    private int addedGlobalsOffset;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.model.component;
 import java.util.Collection;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleIdentifier;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;