added documentation for EntityCreationService, RedactionEntity to prompts

.gitlab-ci.yml

@@ -4,33 +4,3 @@ include:
   - project: 'gitlab/gitlab'
     ref: 'main'
     file: 'ci-templates/gradle_java.yml'
-
-deploy:
-  stage: deploy
-  tags:
-    - dind
-  script:
-    - echo "Building with gradle version ${BUILDVERSION}"
-    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
-    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
-  artifacts:
-    reports:
-      dotenv: version.env
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH =~ /^release/
-    - if: $CI_COMMIT_TAG
-
-pages:
-  stage: build
-  tags:
-    - dind
-  script:
-    - echo "Generating Javadoc..."
-    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
-    - mkdir public
-    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
-  artifacts:
-    paths:
-      - public

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -6,15 +6,11 @@ plugins {
     jacoco
 }
 
-group = "com.iqser.red.service"
+group = "com.iqser.red"
 
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
-pmd {
-    isConsoleOutput = true
-}
-
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }
@@ -43,6 +39,25 @@ tasks.jacocoTestReport {
     }
 }
 
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
 java {
     withJavadocJar()
 }
@@ -57,4 +72,4 @@ repositories {
             password = providers.gradleProperty("mavenPassword").getOrNull();
         }
     }
-}
+}

config/pmd/pmd.xml

@@ -9,12 +9,13 @@
     </description>
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
         <exclude name="AssignmentInOperand"/>
-        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

config/pmd/test_pmd.xml

@@ -10,13 +10,15 @@
 
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
         <exclude name="AssignmentInOperand"/>
         <exclude name="TestClassWithoutTestCases"/>
-        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

drools-prompt/EntityCreationService_doc

@@ -0,0 +1,124 @@
+/**
+* Searches the provided SemanticNode for the keyword and creates an Entity for each occurrence.
+* @param keyword the string to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as byString, but case insensitive.
+*/
+public Stream<RedactionEntity> byStringIgnoreCase(String keyword, String type, EntityType entityType, SemanticNode node)
+/**
+* Searches the provided SemanticNode with the regexPattern and creates a new RedactionEntity with the provided group for each occurrence.
+* @param regexPattern The regexPattern
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param group the regexPattern group, that should be the entity
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegex, but case insensitive.
+*/
+public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegex, but can handle patterns with linebreaks.
+*/
+public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegexWithLineBreaks, but case insensitive.
+*/
+public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Finds the provided string, and creates a new RedactionEntity from the text after until the end of the line it is found in.
+* @param string The keyword to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as lineAfterString, but with multiple keywords
+*/
+public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node)
+/**
+* Finds the provided string in a TableCell, and creates a new RedactionEntity in the same line but adjacent table cells to the right.
+* @param string The keyword to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param table The TableNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, TableNode table)
+/**
+* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
+*
+* @param boundary   The boundary of the redaction entity.
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node       The semantic node where the boundary is.
+* @return An Optional containing the new redaction entity.
+*/
+public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node)
+/**
+* Creates new RedactionEntities between the provided start and stop boundaries. The start and stop boundaries are excluded.
+* If any boundaries of the new RedactionEntities overlap, only the shortest boundary will be used.
+* @param startBoundaries List of start boundaries
+* @param stopBoundaries List of stop boundaries
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node       The semantic node where the boundaries are.
+* @return A Stream of new RedactionEntities between the start and stop boundaries
+*/
+public Stream<RedactionEntity> betweenBoundaries(List<Boundary> startBoundaries, List<Boundary> stopBoundaries, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenBoundaries, but it creates the start and stop boundaries by performing a text search on the provided SemanticNode.
+*/
+public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenStrings, but case insensitive.
+*/
+public Stream<RedactionEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* These 6 functions work the same as betweenStrings, but they also include the start and/or stop strings or are case insensitive, depending on their name.
+*/
+public Stream<RedactionEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenBoundaries, but it creates the start and stop boundaries by performing a regex search on the provided SemanticNode.
+*/
+public Stream<RedactionEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenRegexes, but case insensitive.
+*/
+public Stream<RedactionEntity> betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node)
+/**
+* Creates a new RedactionEntity which has the same boundary as the provided SemanticNode.
+* @param node The SemanticNode to create a new RedactionEntity from.
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @return An optional RedactionEntity. Is empty, if the provided SemanticNode is empty.
+*/
+public Optional<RedactionEntity> bySemanticNode(SemanticNode node, String type, EntityType entityType)
+/**
+* Same as bySemanticNode, but ignores the SemanticNode, if its not a Paragraph and all its child SemanticNodes, that are not Paragraphs.
+*/
+public Stream<RedactionEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType)
+/**
+* Searches the provided SemanticNode for the provided string, and creates a new RedactionEntity, from the end of the first occurrence of the string until the end of the SemanticNode.
+* @param string The string to search for
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node The SemanticNode to use and search in
+* @return An optional RedactionEntity, is empty, if the SemanticNode is empty, or the string isn't found in the SemanticNode.
+*/
+public Optional<RedactionEntity> semanticNodeAfterString(String string, String type, EntityType entityType, SemanticNode node)

drools-prompt/Page_doc

@@ -0,0 +1,18 @@
+/**
+ * Retrieves the main body text block.
+ *
+ * @return The text block representing the main body of the document.
+ */
+public TextBlock getMainBodyTextBlock()
+/**
+* Gets all Entities located on the page
+*
+* @return Set of all Entities associated with this Page
+*/
+Set<RedactionEntity> getEntities();
+/**
+* Returns the Page Number
+*
+* @return The number of this page
+*/
+Integer getPageNumber();

drools-prompt/RedactionEntity_doc

@@ -0,0 +1,33 @@
+/**
+* Sets the Entity to applied, this is the default.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void apply(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Same as apply, but legalBasis can be null.
+*/
+void force(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Sets the Entity to not applied.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void skip(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Sets the Entity to ignored, is preferred to remove in most cases.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void ignore(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Removes the entity entirely, also removes it from all EntitySets and sets intersectingNodes and deepestFullyContainingNode to null.
+* Should only be used in a few cases!
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void remove(String ruleIdentifier, String reason, String legalBasis)

drools-prompt/RedactionEntity_properties_doc

@@ -0,0 +1,39 @@
+/**
+* A boundary has a start and end property accessible by start() and end() respectively. It marks the start and end offset in String offsets.
+*/
+Boundary boundary
+/**
+* The type of an Entity identifies groups of Entities. Examples for Entites we want to find are "CBI_author" for authors, "CBI_address" for addresses or "PII" for personally identifiable information.
+* Other types include helper types, which interact with the main Entities, for example "published_information" or "vertebrate". These Entities are used to modify the main Entities, if they occur in the same Section.
+* A typical example would be to ignore Entities of type "CBI_author", if they occur in the same Section as "published_information" entities.
+*/
+String type
+/**
+* The EntityType can be one of four different values: ENTITY, RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION.
+* If an ENTITY is overlapped by a FALSE_POSITIVE, the ENTITY is removed. If a RECOMMENDATION is overlapped by either an ENTITY or FALSE_RECOMMENDATION, it is removed.
+*/
+EntityType entityType
+/**
+* The text the Entity represents.
+*/
+String value
+/**
+* Up to three words after the Entity in the text.
+*/
+String textAfter
+/**
+* Up to three words before the Entity in the text.
+*/
+String textBefore
+/**
+* All pages whose TextBlock intersects the boundary of this entity. Is always equal to the Pages which have this RedactionEntity in their EntitySet.
+*/
+Set<Page> pages
+/**
+* All SemanticNodes whose TextBlock intersects the boundary of this entity. Is always equal to the SemanticNodes which have this RedactionEntity in their EntitySet.
+*/
+List<SemanticNode> intersectingNodes
+/**
+* The SemanticNode which is the deepest in the Tree structure and whose TextBlock fully contains the boundary of this Node.
+*/
+SemanticNode deepestFullyContainingNode

drools-prompt/Section_doc

@@ -0,0 +1,6 @@
+/**
+* Determines whether this Section has any tables.
+*
+* @return {@code true} if there are tables, {@code false} otherwise
+*/
+public boolean hasTables()

drools-prompt/SemanticNode_doc

@@ -0,0 +1,181 @@
+/**
+* Returns the type of this node, such as NodeType.SECTION, NodeType.PARAGRAPH, etc.
+*
+* @return NodeType of this node
+*/
+NodeType getType();
+/**
+* Any Node maintains its own Set of Entities.
+* This Set contains all Entities whose boundary intersects the boundary of this node.
+* The Entities might overlap with the Entities in other Sets
+*
+* @return Set of all Entities associated with this Node
+*/
+Set<RedactionEntity> getEntities();
+/**
+* Returns all Pages this SemanticNode is associated with.
+*
+* @return Set of Pages this node appears on.
+*/
+Set<Page> getPages()
+/**
+* Checks if this node appears on the specified page number.
+*
+* @param pageNumber The page number to check.
+* @return True if this node is found on the specified page number, false otherwise.
+*/
+boolean isOnPage(int pageNumber)
+/**
+* Returns the closest Headline associated with this SemanticNode
+*
+* @return First Headline found.
+*/
+Headline getHeadline()
+/**
+* @return The SemanticNode representing the Parent in the DocumentTree
+* throws NotFoundException, when no parent is present
+*/
+SemanticNode getParent()
+/**
+* Checks whether this SemanticNode has any Entity of the provided type.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type string representing the type of entity to check for
+* @return true, if this SemanticNode has at least one Entity of the provided type
+*/
+boolean hasEntitiesOfType(String type)
+/**
+* Checks whether this SemanticNode has any Entity of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types an array of strings representing the types of entities to check for
+* @return true, if this SemanticNode has at least one Entity of any of the provided types
+*/
+boolean hasEntitiesOfAnyType(String... types)
+/**
+* Checks whether this SemanticNode has at least one Entity of each of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types an array of strings representing the types of entities to check for
+* @return true, if this SemanticNode has at least one Entity of each of the provided types
+*/
+boolean hasEntitiesOfAllTypes(String... types)
+/**
+* Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type string representing the type of entities to return
+* @return List of RedactionEntities of any the type
+*/
+List<RedactionEntity> getEntitiesOfType(String type)
+/**
+* Returns a List of Entities in this SemanticNode which have any of the provided types such as "CBI_author".
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types A list of strings representing the types of entities to return
+* @return List of RedactionEntities of any provided type
+*/
+List<RedactionEntity> getEntitiesOfType(List<String> types)
+/**
+* Returns a List of Entities in this SemanticNode which have any of the provided types.
+* Ignores Entity with the ignored flag set to true or the removed flag set to true.
+*
+* @param types A list of strings representing the types of entities to return
+* @return List of RedactionEntities that match any of the provided types
+*/
+List<RedactionEntity> getEntitiesOfType(String... types)
+/**
+* Checks whether this SemanticNode contains the provided String.
+*
+* @param string A String which the TextBlock might contain
+* @return true, if this node's TextBlock contains the string
+*/
+boolean containsString(String string)
+/**
+* Checks whether this SemanticNode contains all the provided Strings.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains all strings
+*/
+boolean containsAllStrings(String... strings)
+/**
+* Checks whether this SemanticNode contains any of the provided Strings.
+*
+* @param strings A List of Strings to check if they are contained in the TextBlock
+* @return true, if this node's TextBlock contains any of the provided strings
+*/
+boolean containsAnyString(String... strings)
+/**
+* Checks whether this SemanticNode contains all the provided Strings ignoring case.
+*
+* @param string A String which the TextBlock might contain
+* @return true, if this node's TextBlock contains the string ignoring case
+*/
+boolean containsStringIgnoreCase(String string)
+/**
+* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains any of the strings
+*/
+boolean containsAnyStringIgnoreCase(String... strings)
+/**
+* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains any of the strings
+*/
+boolean containsAllStringsIgnoreCase(String... strings)
+/**
+* Checks whether this SemanticNode matches the provided regex pattern.
+*
+* @param regexPattern A String representing a regex pattern, which the TextBlock might contain
+* @return true, if this node's TextBlock contains the regex pattern
+*/
+boolean matchesRegex(String regexPattern)
+/**
+* Checks whether this SemanticNode matches the provided regex pattern ignoring case.
+*
+* @param regexPattern A String representing a regex pattern, which the TextBlock might contain
+* @return true, if this node's TextBlock contains the regex pattern ignoring case
+*/
+boolean matchesRegexIgnoreCase(String regexPattern)
+/**
+* Streams all children located directly underneath this node in the DocumentTree.
+*
+* @return Stream of all children
+*/
+Stream<SemanticNode> streamChildren()
+/**
+* Streams all children located directly underneath this node in the DocumentTree of the provided type.
+*
+* @param nodeType the type of nodes to stream
+* @return Stream of all children of the provided type
+*/
+Stream<SemanticNode> streamChildrenOfType(NodeType nodeType)
+/**
+* Recursively streams all SemanticNodes located underneath this node in the DocumentTree in order.
+*
+* @return Stream of all SubNodes
+*/
+Stream<SemanticNode> streamAllSubNodes()
+/**
+* Recursively streams all SemanticNodes of a specified type located underneath this node in the DocumentTree in order.
+*
+* @param nodeType the type of nodes to be streamed
+* @return a Stream of all SubNodes of the specified type
+*/
+Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType)
+/**
+* The Boundary is the start and end string offsets in the reading order of the document.
+*
+* @return Boundary of this Node's TextBlock
+*/
+Boundary getBoundary()
+/**
+ * The SectionIdentifier uses the numeric identifiers of Headlines to infer a tree structure.
+ * It implements functions such as sectionIdentifier.isChildOf(otherSectionIdentifier) and sectionIdentifier.isParentOf(otherSectionIdentifier)
+ *
+ * @return The SectionIdentifier from the first Headline.
+ */
+SectionIdentifier getSectionIdentifier()

drools-prompt/Table_doc

@@ -0,0 +1,77 @@
+/**
+* Streams all entities in this table, that appear in a row, which contains any of the provided strings.
+*
+* @param strings Strings to check whether a row contains them
+* @return Stream of all entities in this table, that appear in a row, which contains any of the provided strings
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) 
+/**
+* Checks whether the specified row contains all the provided strings.
+*
+* @param row     the row to check as an Integer, must be smaller than numberOfRows
+* @param strings a list of strings to check for
+* @return true, if all strings appear in the provided row
+*/
+boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) 
+/**
+* Streams all entities which appear in a row where at least one cell has the provided header and the provided value.
+*
+* @param header the header value to search for
+* @param value  the string which the table cell should contain
+* @return a stream of all entities, which appear in a row where at least one cell has the provided header and the provided value.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) 
+/**
+* Streams all entities which appear in a row where at least one cell has the provided header and any provided value.
+*
+* @param header the header value to search for
+* @param values the strings which the table cell should contain
+* @return a stream of all entities, which appear in a row where at least one cell has the provided header and any provided value.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) 
+/**
+* Streams all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types type strings to check whether a row contains an entity like them
+* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) 
+/**
+* Streams all entities in this table, that appear in a row, which contains no entity of any of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types type strings to check whether a row contains an entity like them
+* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) 
+/**
+* Streams all TableCells in this Table which have the provided header row-wise.
+*
+* @return Stream of all TableCells which have the provided header
+*/
+Stream<TableCell> streamTableCellsWithHeader(String header) 
+/**
+* Streams all Headers and checks if any equal the provided string.
+*
+* @param header string to check the headers for
+* @return true, if at least one header equals the provided string
+*/
+boolean hasHeader(String header) 
+/**
+* Checks if this table has a column with the provided header and any of the table cells in that column contain the provided value.
+*
+* @param header string to find header cells
+* @param value  string to check cells with provided header
+* @return true, if this table has a column with the provided header and any of the table cells in that column contain the provided value
+*/
+boolean hasRowWithHeaderAndValue(String header, String value) 
+/**
+* Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type            the type of entities to search for
+* @param redactionEntity the entity, which appears in the row to search
+* @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
+*/
+List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) 

drools-prompt/drools-prompt

@@ -0,0 +1,591 @@
+From now on, you are a Drools rule generator.
+
+You have a Document data structure written in Java with the following objects:
+
+- Section
+- Table
+- TableCell
+- Paragraph
+- Headline
+- Page
+- RedactionEntity
+- EntityCreationService
+
+The Section, Table, TableCell, Paragraph, and Headline implement a common interface called SemanticNode. SemanticNodes are arranged in a tree-like fashion, where any SemanticNode can have multiple SemanticNodes as children. The arrangement is as follows:
+- Tables only have TableCells as children.
+- TableCells may have any child, except TableCells.
+- Paragraphs and Headlines have no children.
+- Sections may have any child except TableCells, but if it contains Paragraphs as well as Tables, it is split into a Section with multiple Sections as children, where any child Section only contains either Tables or Paragraphs.
+   Further, if the first SemanticNode is a Headline it remains the first child in the Parent Section, before any subsections.
+
+The goal of the Software is to find pieces of Text that are relevant. Each piece of text is represented by a RedactionEntity.
+The main pieces of relevant text are Text we want to redact. For example, we want to redact all Authors of a dossier. Or all personally identifiable information, such as E-Mails and Telephone Numbers.
+RedactionEntities may also represent other pieces of text, such as published information, or certain species of vertebrates.
+The RedactionEntities are part of the document structure, such that they are referenced in each SemanticNode and Page, which contains it, and further, the RedactionEntity references each Page and SemanticNode it occurs in.
+So the same RedactionEntity occurs in the paragraph it is located, as well as all its parent Sections.
+
+Previous to the execution of rules, the Document structure is assembled and a text search is performed to create initial Entities of different types. They are then inserted into the document structure.
+Then the KieSession is created and each SemanticNode and Entity is inserted into its working memory.
+----------------------------------------------------------------
+The relevant functions for SemanticNode:
+/**
+* Returns the type of this node, such as NodeType.SECTION, NodeType.PARAGRAPH, etc.
+*
+* @return NodeType of this node
+*/
+NodeType getType();
+
+/**
+* Any Node maintains its own Set of Entities.
+* This Set contains all Entities whose boundary intersects the boundary of this node.
+* The Entities might overlap with the Entities in other Sets
+*
+* @return Set of all Entities associated with this Node
+*/
+Set<RedactionEntity> getEntities();
+
+/**
+* Returns all Pages this SemanticNode is associated with.
+*
+* @return Set of Pages this node appears on.
+*/
+Set<Page> getPages()
+
+/**
+* Checks if this node appears on the specified page number.
+*
+* @param pageNumber The page number to check.
+* @return True if this node is found on the specified page number, false otherwise.
+*/
+boolean onPage(int pageNumber)
+
+/**
+* For Sections it searches its children and returns the first Headline.
+* For Paragraphs, Tables, and TableCells it returns getHeadline() of getParent()
+* For Headline it returns itself and for Headers or Footers it returns an empty dummy Headline.
+*
+* @return First Headline found.
+*/
+Headline getHeadline()
+
+/**
+* @return The SemanticNode representing the Parent in the DocumentTree
+* When no parent is present, the Document is returned. And for the Document itself it throws an UnsupportedOperationException.
+*/
+SemanticNode getParent()
+
+/**
+* Checks whether this SemanticNode has any Entity of the provided type.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type string representing the type of entity to check for
+* @return true, if this SemanticNode has at least one Entity of the provided type
+*/
+boolean hasEntitiesOfType(String type)
+
+/**
+* Checks whether this SemanticNode has any Entity of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types an array of strings representing the types of entities to check for
+* @return true, if this SemanticNode has at least one Entity of any of the provided types
+*/
+boolean hasEntitiesOfAnyType(String... types)
+
+/**
+* Checks whether this SemanticNode has at least one Entity of each of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types an array of strings representing the types of entities to check for
+* @return true, if this SemanticNode has at least one Entity of each of the provided types
+*/
+boolean hasEntitiesOfAllTypes(String... types)
+
+/**
+* Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type string representing the type of entities to return
+* @return List of RedactionEntities of any the type
+*/
+List<RedactionEntity> getEntitiesOfType(String type)
+
+/**
+* Returns a List of Entities in this SemanticNode which have any of the provided types such as "CBI_author".
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types A list of strings representing the types of entities to return
+* @return List of RedactionEntities of any provided type
+*/
+List<RedactionEntity> getEntitiesOfType(List<String> types)
+
+/**
+* Returns a List of Entities in this SemanticNode which have any of the provided types.
+* Ignores Entity with the ignored flag set to true or the removed flag set to true.
+*
+* @param types A list of strings representing the types of entities to return
+* @return List of RedactionEntities that match any of the provided types
+*/
+List<RedactionEntity> getEntitiesOfType(String... types)
+
+/**
+* Checks whether this SemanticNode contains the provided String.
+*
+* @param string A String which the TextBlock might contain
+* @return true, if this node's TextBlock contains the string
+*/
+boolean containsString(String string)
+
+/**
+* Checks whether this SemanticNode contains all the provided Strings.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains all strings
+*/
+boolean containsAllStrings(String... strings)
+
+/**
+* Checks whether this SemanticNode contains any of the provided Strings.
+*
+* @param strings A List of Strings to check if they are contained in the TextBlock
+* @return true, if this node's TextBlock contains any of the provided strings
+*/
+boolean containsAnyString(String... strings)
+/**
+* Checks whether this SemanticNode contains all the provided Strings ignoring case.
+*
+* @param string A String which the TextBlock might contain
+* @return true, if this node's TextBlock contains the string ignoring case
+*/
+boolean containsStringIgnoreCase(String string)
+
+/**
+* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains any of the strings
+*/
+boolean containsAnyStringIgnoreCase(String... strings)
+
+/**
+* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+*
+* @param strings A List of Strings which the TextBlock might contain
+* @return true, if this node's TextBlock contains any of the strings
+*/
+boolean containsAllStringsIgnoreCase(String... strings)
+
+/**
+* Checks whether this SemanticNode matches the provided regex pattern.
+*
+* @param regexPattern A String representing a regex pattern, which the TextBlock might contain
+* @return true, if this node's TextBlock contains the regex pattern
+*/
+boolean matchesRegex(String regexPattern)
+
+/**
+* Checks whether this SemanticNode matches the provided regex pattern ignoring case.
+*
+* @param regexPattern A String representing a regex pattern, which the TextBlock might contain
+* @return true, if this node's TextBlock contains the regex pattern ignoring case
+*/
+boolean matchesRegexIgnoreCase(String regexPattern)
+
+/**
+* Streams all children located directly underneath this node in the DocumentTree.
+*
+* @return Stream of all children
+*/
+Stream<SemanticNode> streamChildren()
+
+/**
+* Streams all children located directly underneath this node in the DocumentTree of the provided type.
+*
+* @param nodeType the type of nodes to stream
+* @return Stream of all children of the provided type
+*/
+Stream<SemanticNode> streamChildrenOfType(NodeType nodeType)
+
+/**
+* Recursively streams all SemanticNodes located underneath this node in the DocumentTree in order.
+*
+* @return Stream of all SubNodes
+*/
+Stream<SemanticNode> streamAllSubNodes()
+
+/**
+* Recursively streams all SemanticNodes of a specified type located underneath this node in the DocumentTree in order.
+*
+* @param nodeType the type of nodes to be streamed
+* @return a Stream of all SubNodes of the specified type
+*/
+Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType)
+
+/**
+* The Boundary is the start and end string offsets in the reading order of the document.
+*
+* @return Boundary of this Node's TextBlock
+*/
+Boundary getBoundary()
+
+/**
+ * The SectionIdentifier uses the numeric identifiers of Headlines to infer a tree structure.
+ * It implements functions such as sectionIdentifier.isChildOf(otherSectionIdentifier) and sectionIdentifier.isParentOf(otherSectionIdentifier)
+ *
+ * @return The SectionIdentifier from the first Headline.
+ */
+SectionIdentifier getSectionIdentifier()
+
+----------------------------------------------------------------
+The Table has the additional functions:
+/**
+* Streams all entities in this table, that appear in a row, which contains any of the provided strings.
+*
+* @param strings Strings to check whether a row contains them
+* @return Stream of all entities in this table, that appear in a row, which contains any of the provided strings
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings)
+
+/**
+* Checks whether the specified row contains all the provided strings.
+*
+* @param row     the row to check as an Integer, must be smaller than numberOfRows
+* @param strings a list of strings to check for
+* @return true, if all strings appear in the provided row
+*/
+boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings)
+
+/**
+* Streams all entities which appear in a row where at least one cell has the provided header and the provided value.
+*
+* @param header the header value to search for
+* @param value  the string which the table cell should contain
+* @return a stream of all entities, which appear in a row where at least one cell has the provided header and the provided value.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value)
+
+/**
+* Streams all entities which appear in a row where at least one cell has the provided header and any provided value.
+*
+* @param header the header value to search for
+* @param values the strings which the table cell should contain
+* @return a stream of all entities, which appear in a row where at least one cell has the provided header and any provided value.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values)
+
+/**
+* Streams all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types type strings to check whether a row contains an entity like them
+* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types)
+
+/**
+* Streams all entities in this table, that appear in a row, which contains no entity of any of the provided types.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param types type strings to check whether a row contains an entity like them
+* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+*/
+Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types)
+
+/**
+* Streams all TableCells in this Table which have the provided header row-wise.
+*
+* @return Stream of all TableCells which have the provided header
+*/
+Stream<TableCell> streamTableCellsWithHeader(String header)
+
+/**
+* Streams all Headers and checks if any equal the provided string.
+*
+* @param header string to check the headers for
+* @return true, if at least one header equals the provided string
+*/
+boolean hasHeader(String header)
+
+/**
+* Checks if this table has a column with the provided header and any of the table cells in that column contain the provided value.
+*
+* @param header string to find header cells
+* @param value  string to check cells with provided header
+* @return true, if this table has a column with the provided header and any of the table cells in that column contain the provided value
+*/
+boolean hasRowWithHeaderAndValue(String header, String value)
+
+/**
+* Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
+* Ignores Entity with ignored == true or removed == true.
+*
+* @param type            the type of entities to search for
+* @param redactionEntity the entity, which appears in the row to search
+* @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
+*/
+List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity)
+----------------------------------------------------------------
+The Section has these additional Rules:
+/**
+* Determines whether this Section has any tables.
+*
+* @return {@code true} if there are tables, {@code false} otherwise
+*/
+boolean hasTables()
+----------------------------------------------------------------
+The Page Object has the following functions:
+/**
+* Retrieves the main body text block.
+* @return The text block representing the main body of the document.
+*/
+public TextBlock getMainBodyTextBlock()
+/**
+* @return All SemanticNodes that occur on the page, except Header and Footer
+*/
+public List<SemanticNode> getMainBody()
+/**
+* Gets all Entities located on the page
+* @return Set of all Entities associated with this Page
+*/
+Set<RedactionEntity> getEntities();
+/**
+* Returns the Page Number
+*
+* @return The number of this page
+*/
+Integer getPageNumber();
+----------------------------------------------------------------
+
+----------------------------------------------------------------
+The RedactionEntity has the following properties:
+/**
+* A boundary has a start and end property accessible by start() and end() respectively. It marks the start and end offset in String offsets.
+*/
+Boundary boundary
+/**
+* The type of an Entity identifies groups of Entities. Examples for Entites we want to find are "CBI_author" for authors, "CBI_address" for addresses or "PII" for personally identifiable information.
+* Other types include helper types, which interact with the main Entities, for example "published_information" or "vertebrate". These Entities are used to modify the main Entities, if they occur in the same Section.
+* A typical example would be to ignore Entities of type "CBI_author", if they occur in the same Section as "published_information" entities.
+*/
+String type
+/**
+* The EntityType can be one of four different values: ENTITY, RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION.
+* If an ENTITY is overlapped by a FALSE_POSITIVE, the ENTITY is removed. If a RECOMMENDATION is overlapped by either an ENTITY or FALSE_RECOMMENDATION, it is removed.
+*/
+EntityType entityType
+/**
+* The text the Entity represents.
+*/
+String value
+/**
+* Up to three words after the Entity in the text.
+*/
+String textAfter
+/**
+* Up to three words before the Entity in the text.
+*/
+String textBefore
+/**
+* All pages whose TextBlock intersects the boundary of this entity. Is always equal to the Pages which have this RedactionEntity in their EntitySet.
+*/
+Set<Page> pages
+/**
+* All SemanticNodes whose TextBlock intersects the boundary of this entity. Is always equal to the SemanticNodes which have this RedactionEntity in their EntitySet.
+*/
+List<SemanticNode> intersectingNodes
+/**
+* The SemanticNode which is the deepest in the Tree structure and whose TextBlock fully contains the boundary of this Node.
+*/
+SemanticNode deepestFullyContainingNode
+
+The RedactionEntity also has the following methods:
+/**
+* Sets the Entity to applied, this is the default.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void apply(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Same as apply, but legalBasis can be null.
+*/
+void force(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Sets the Entity to not applied.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void skip(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Sets the Entity to ignored, is preferred to remove in most cases.
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void ignore(String ruleIdentifier, String reason, String legalBasis)
+/**
+* Removes the entity entirely, also removes it from all EntitySets and sets intersectingNodes and deepestFullyContainingNode to null.
+* Should only be used in a few cases!
+* @param ruleIdentifier Should always be equal to the ruleIdentifier in the Rule name
+* @param reason Should describe the intention of the rule in a few words
+* @param legalBasis Is dependent on the rule, if none is known the default is "n-a", can't be null
+*/
+void remove(String ruleIdentifier, String reason, String legalBasis)
+----------------------------------------------------------------
+The EntityCreationService offers the following functions:
+/**
+* Searches the provided SemanticNode for the keyword and creates an Entity for each occurrence.
+* @param keyword the string to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as byString, but case insensitive.
+*/
+public Stream<RedactionEntity> byStringIgnoreCase(String keyword, String type, EntityType entityType, SemanticNode node)
+/**
+* Searches the provided SemanticNode with the regexPattern and creates a new RedactionEntity with the provided group for each occurrence.
+* @param regexPattern The regexPattern
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param group the regexPattern group, that should be the entity
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegex, but case insensitive.
+*/
+public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegex, but can handle patterns with linebreaks.
+*/
+public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Same as byRegexWithLineBreaks, but case insensitive.
+*/
+public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node)
+/**
+* Finds the provided string, and creates a new RedactionEntity from the text after until the end of the line it is found in.
+* @param string The keyword to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param node The SemanticNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as lineAfterString, but with multiple keywords
+*/
+public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node)
+/**
+* Finds the provided string in a TableCell, and creates a new RedactionEntity in the same line but adjacent table cells to the right.
+* @param string The keyword to search for
+* @param type The type of the RedactionEntity to be created
+* @param entityType The EntityType of the RedactionEntity to be created
+* @param table The TableNode to search in
+* @return A Stream of RedactionEntities with the keyword as value, the type as type and the provided EntityType
+*/
+public Stream<RedactionEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, TableNode table)
+/**
+* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
+*
+* @param boundary   The boundary of the redaction entity.
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node       The semantic node where the boundary is.
+* @return An Optional containing the new redaction entity.
+*/
+public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node)
+/**
+* Creates new RedactionEntities between the provided start and stop boundaries. The start and stop boundaries are excluded.
+* If any boundaries of the new RedactionEntities overlap, only the shortest boundary will be used.
+* @param startBoundaries List of start boundaries
+* @param stopBoundaries List of stop boundaries
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node       The semantic node where the boundaries are.
+* @return A Stream of new RedactionEntities between the start and stop boundaries
+*/
+public Stream<RedactionEntity> betweenBoundaries(List<Boundary> startBoundaries, List<Boundary> stopBoundaries, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenBoundaries, but it creates the start and stop boundaries by performing a text search on the provided SemanticNode.
+*/
+public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenStrings, but case insensitive.
+*/
+public Stream<RedactionEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* These 6 functions work the same as betweenStrings, but they also include the start and/or stop strings or are case insensitive, depending on their name.
+*/
+public Stream<RedactionEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node)
+public Stream<RedactionEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenBoundaries, but it creates the start and stop boundaries by performing a regex search on the provided SemanticNode.
+*/
+public Stream<RedactionEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node)
+/**
+* Same as betweenRegexes, but case insensitive.
+*/
+public Stream<RedactionEntity> betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node)
+/**
+* Creates a new RedactionEntity which has the same boundary as the provided SemanticNode.
+* @param node The SemanticNode to create a new RedactionEntity from.
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @return An optional RedactionEntity. Is empty, if the provided SemanticNode is empty.
+*/
+public Optional<RedactionEntity> bySemanticNode(SemanticNode node, String type, EntityType entityType)
+/**
+* Same as bySemanticNode, but ignores the SemanticNode, if its not a Paragraph and all its child SemanticNodes, that are not Paragraphs.
+*/
+public Stream<RedactionEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType)
+/**
+* Searches the provided SemanticNode for the provided string, and creates a new RedactionEntity, from the end of the first occurrence of the string until the end of the SemanticNode.
+* @param string The string to search for
+* @param type       The type of the redaction entity.
+* @param entityType The entity type of the redaction entity.
+* @param node The SemanticNode to use and search in
+* @return An optional RedactionEntity, is empty, if the SemanticNode is empty, or the string isn't found in the SemanticNode.
+*/
+public Optional<RedactionEntity> semanticNodeAfterString(String string, String type, EntityType entityType, SemanticNode node)
+----------------------------------------------------------------
+Rules may be grouped into two categories.
+The first category changes existing RedactionEntities, and the second creates new RedactionEntities.
+
+There are two different types of rules, one you create new Entities and in the other you change or remove existing Entities.
+An Entity is any piece of text, uniquely identified in the Document by its Boundary, its Type and its EntityType. The Boundary consists of a start and stop index in the text of the document.
+The Type is a String like "PII", which stands for
+The goal is to find entities that fulfill certain conditions. Each SemanticNode has its own set of entities, but these sets may have intersections.
+For example, a Section contains all the entities in any of its children. Additionally, if an entity overlaps two SemanticNodes, both paragraphs have this entity in their sets.
+
+To generate Drools rules for the scenario of changing or updating Entities, consider the following information:
+
+1. Conditions: Specify the conditions that must be met for an entity to be selected. For example:
+- The entity has a specific attribute value.
+- The entity is within a certain range of values.
+- The entity satisfies a complex combination of conditions.
+
+2. Actions: Define the actions to be performed when an entity fulfills the conditions. This could include:
+- Adding the entity to a result set.
+- Modifying the entity's attributes.
+- Triggering some other behavior or logic.
+
+3. Rule Structure: Determine the structure of the Drools rules. This typically consists of:
+- Rule names: Choose meaningful names for your rules.
+- Rule attributes: Set the salience (priority) of rules if necessary.
+- Conditions: Define the conditions based on the requirements.
+- Actions: Specify the actions to be performed when the conditions are met.
+
+Remember to provide specific examples, use case scenarios, and any additional requirements you have for the Drools rules.
+
+Please provide any specific conditions, actions, or examples that you would like to be incorporated into the Drools rules.

publish-custom-image.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-dir=${PWD##*/}
-gradle assemble
-
-# Get the current Git branch
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-# Get the short commit hash (first 5 characters)
-commit_hash=$(git rev-parse --short=5 HEAD)
-
-# Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
-
-gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
-echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -1,28 +1,14 @@
+
 plugins {
     id("com.iqser.red.service.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.1.0"
 }
 
 description = "redaction-service-api-v1"
 
 dependencies {
-    implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
+    implementation("org.springframework:spring-web:6.0.11")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725")
 }
 
-publishing {
-    publications {
-        create<MavenPublication>(name) {
-            from(components["java"])
-        }
-    }
-    repositories {
-        maven {
-            url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
-            credentials {
-                username = providers.gradleProperty("mavenUser").getOrNull();
-                password = providers.gradleProperty("mavenPassword").getOrNull();
-            }
-        }
-    }
-}
+

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeResponse {
-
-    private String fileId;
-    private List<UnprocessedManualEntity> unprocessedManualEntities;
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxDeprecatedWarnings {
-
-    Integer line;
-    Integer column;
-    String message;
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxErrorMessage {
-
-    Integer line;
-    Integer column;
-    String message;
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -1,36 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxValidation {
-
-    @Builder.Default
-    List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
-    @Builder.Default
-    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
-
-
-    public void addErrorMessage(int line, int column, String message) {
-
-        getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build());
-    }
-
-    public boolean isCompiled() {
-
-        return droolsSyntaxErrorMessages.isEmpty();
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,21 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigrationRequest {
-
-    String dossierTemplateId;
-    String dossierId;
-    String fileId;
-    ManualRedactions manualRedactions;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationResponse.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigrationResponse {
-
-    String dossierId;
-    String fileId;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public class QueueNames {
-
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-    public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
-
-    public static final String MIGRATION_QUEUE = "migrationQueue";
-    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
-    public static final String MIGRATION_DLQ = "migrationDLQ";
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleValidationModel.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class RuleValidationModel {
-
-    String ruleFileType;
-    String rulesString;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -1,27 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-@EqualsAndHashCode
-public class UnprocessedManualEntity {
-
-    private String annotationId;
-    private String textBefore;
-    private String textAfter;
-    private String section;
-    @Builder.Default
-    private List<Position> positions = new ArrayList<>();
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -4,12 +4,9 @@ import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
-import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
-
 public interface RedactionResource {
 
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
-    DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel);
+    void testRules(@RequestBody String rules);
 
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -3,43 +3,38 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
     application
     id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.1.5"
-    id("io.spring.dependency-management") version "1.1.3"
-    id("org.sonarqube") version "4.4.1.3373"
-    id("io.freefair.lombok") version "8.4"
+    id("org.springframework.boot") version "3.1.2"
+    id("io.spring.dependency-management") version "1.1.2"
+    id("org.sonarqube") version "4.3.0.3225"
+    id("io.freefair.lombok") version "8.1.0"
 }
 
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.94.0"
+val layoutParserVersion = "0.23.0"
 val jacksonVersion = "2.15.2"
-val droolsVersion = "9.44.0.Final"
-val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.360.0"
-val springBootStarterVersion = "3.1.5"
+val droolsVersion = "8.42.0.Final"
+val pdfBoxVersion = "3.0.0-alpha2"
 
 configurations {
     all {
-        exclude(group = "commons-logging", module = "commons-logging")
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
-        exclude(group = "com.iqser.red.commons", module = "logging-commons")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
     }
 }
 
 dependencies {
 
     implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.119.0") { exclude(group = "org.springframework.boot") }
     implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
 
-    implementation("com.iqser.red.commons:spring-commons:6.2.0")
+    implementation("com.iqser.red.commons:spring-commons:2.6.0")
     implementation("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.iqser.red.commons:storage-commons:2.24.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -53,21 +48,12 @@ dependencies {
     implementation("org.locationtech.jts:jts-core:1.19.0")
 
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
 
-    implementation("net.logstash.logback:logstash-logback-encoder:7.4")
-    implementation("ch.qos.logback:logback-classic")
-
-    implementation("org.reflections:reflections:0.10.2")
-
-    testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
-    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
-    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.2")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",
@@ -80,16 +66,11 @@ tasks.test {
     configure<JacocoTaskExtension> {
         excludes = listOf("org/drools/**/*")
     }
-    minHeapSize = "512m"
-    maxHeapSize = "1024m"
 }
 
+
 tasks.named<BootBuildImage>("bootBuildImage") {
-
-    environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
-    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8")
-
-    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
+    imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
     }
@@ -105,45 +86,5 @@ tasks.named<BootBuildImage>("bootBuildImage") {
             email.set(providers.gradleProperty("mavenEmail").getOrNull())
             url.set("https://nexus.knecon.com:5001/")
         }
-
-        val dockerTag = "nexus.knecon.com:5001/red/${project.name}:${project.version}"
-        tags.set(listOf(dockerTag))
-    }
-}
-
-fun parseDroolsImports(droolsFilePath: String): List<String> {
-    val imports = mutableListOf<String>()
-    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
-    val desiredPrefix = "com.iqser.red.service.redaction.v1"
-
-    File(droolsFilePath).forEachLine { line ->
-        importPattern.find(line)?.let { matchResult ->
-            val importPath = matchResult.groupValues[1].trim()
-            if (importPath.startsWith(desiredPrefix)) {
-                val formattedPath = importPath.replace('.', '/')
-                imports.add("$formattedPath.java")
-            }
-        }
-    }
-
-    return imports
-}
-
-val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
-
-tasks.register("generateJavaDoc", Javadoc::class) {
-    dependsOn("compileJava")
-    dependsOn("delombok")
-    classpath = project.sourceSets["main"].runtimeClasspath
-    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
-        include(droolsImports)
-    }
-    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
-
-    options.memberLevel = JavadocMemberLevel.PUBLIC
-    (options as StandardJavadocDocletOptions).apply {
-        header = "Redaction Service ${project.version}"
-        footer = "Redaction Service ${project.version}"
-        title = "API Documentation for Redaction Service ${project.version}"
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -6,22 +6,19 @@ import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.aop.ObservedAspect;
 
-@EnableCaching
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
 @Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
@@ -35,10 +32,6 @@ public class Application {
         SpringApplication.run(Application.class, args);
     }
 
-    @Bean
-    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
-        return new ObservedAspect(observationRegistry);
-    }
 
     @Bean
     public TimedAspect timedAspect(MeterRegistry registry) {
@@ -46,18 +39,10 @@ public class Application {
         return new TimedAspect(registry);
     }
 
-
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
-
-    @Bean(initMethod = "findDeprecatedMethods")
-    public DeprecatedElementsFinder deprecatedElementsFinder() {
-
-        return new DeprecatedElementsFinder();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -1,108 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static java.util.stream.Collectors.groupingBy;
-
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.reflections.Reflections;
-import org.reflections.scanners.Scanners;
-import org.reflections.util.ConfigurationBuilder;
-
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class DeprecatedElementsFinder {
-
-    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
-    private Set<Method> deprecatedMethods;
-    @Getter
-    private Map<String, String> deprecatedMethodsSignaturesMap;
-    private Set<Class<?>> deprecatedClasses;
-    @Getter
-    private SearchImplementation classesSearchImplementation;
-    @Getter
-    private SearchImplementation methodsSearchImplementation;
-
-
-    public void findDeprecatedMethods() {
-
-        deprecatedMethods = new HashSet<>();
-        deprecatedClasses = new HashSet<>();
-        deprecatedMethodsSignaturesMap = new HashMap<>();
-
-        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
-                                                          .setExpandSuperTypes(true)
-                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
-
-        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
-
-        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
-
-        if (!deprecatedClasses.isEmpty()) {
-            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
-                                                                                         .collect(Collectors.toSet())));
-            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
-                                                                           .map(Class::getName)
-                                                                           .collect(Collectors.toSet()), false);
-        }
-
-        if (!deprecatedMethods.isEmpty()) {
-            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
-                                                                           .map(Method::getName)
-                                                                           .collect(Collectors.toSet()), false);
-            deprecatedMethodsSignaturesMap = new HashMap<>();
-            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
-                    .collect(groupingBy(Method::getName));
-            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
-                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
-                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
-                                                                       .stream()
-                                                                       .map(this::getMethodSignature)
-                                                                       .collect(Collectors.joining("\n")));
-            }
-
-        }
-
-        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
-    }
-
-
-    public Set<Method> getDeprecatedMethods() {
-
-        if (this.deprecatedMethods == null) {
-            findDeprecatedMethods();
-        }
-        return this.deprecatedMethods;
-    }
-
-
-    public Set<Class<?>> getDeprecatedClasses() {
-
-        if (this.deprecatedClasses == null) {
-            findDeprecatedMethods();
-        }
-        return this.deprecatedClasses;
-    }
-
-    private String getMethodSignature(Method method) {
-
-        String methodName = method.getName();
-        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
-                .map(Type::getTypeName)
-                .collect(Collectors.joining(", "));
-
-        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java

@@ -1,31 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import java.time.Duration;
-
-import org.springframework.boot.autoconfigure.cache.RedisCacheManagerBuilderCustomizer;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.data.redis.cache.RedisCacheConfiguration;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-
-@Configuration
-public class RedisCachingConfiguration {
-
-    @Bean
-    public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() {
-
-        return (builder) -> builder.withCacheConfiguration("documentDataCache",
-                RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
-    }
-
-
-    @Bean
-    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
-
-        RedisTemplate<String, Object> template = new RedisTemplate<>();
-        template.setConnectionFactory(connectionFactory);
-        return template;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
 import com.iqser.red.commons.spring.ErrorMessage;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -3,11 +3,9 @@ package com.iqser.red.service.redaction.v1.server.controller;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
-import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -17,14 +15,14 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
 
-    private final DroolsSyntaxValidationService droolsSyntaxValidationService;
+    private final DroolsExecutionService droolsExecutionService;
 
 
     @Override
-    public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) {
+    public void testRules(@RequestBody String rules) {
 
         try {
-            return droolsSyntaxValidationService.testRules(rulesValidationModel);
+            droolsExecutionService.testRules(rules);
         } catch (Exception e) {
             throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java

@@ -1,11 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.util.Collections;
-
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
 import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RuleBuilderModelService;
 
 import lombok.RequiredArgsConstructor;
 
@@ -13,16 +12,13 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class RuleBuilderController implements RuleBuilderResource {
 
+    private final RuleBuilderModelService ruleBuilderModelService;
+
 
     @Override
     public RuleBuilderModel getRuleBuilderModel() {
 
-        RuleBuilderModel ruleBuilderModel = new RuleBuilderModel();
-
-        ruleBuilderModel.setWhenClauses(Collections.emptyList());
-        ruleBuilderModel.setThenConditions(Collections.emptyList());
-
-        return ruleBuilderModel;
+        return ruleBuilderModelService.getRuleBuilderModel();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java

@@ -1,6 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document;
-
-import java.io.Serializable;
+package com.iqser.red.service.redaction.v1.server.document.data;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
@@ -11,15 +9,13 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
 @AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentData implements Serializable {
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DocumentData {
 
     DocumentPage[] documentPages;
     DocumentTextData[] documentTextData;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.document.data.mapper;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -6,24 +6,23 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -77,7 +76,6 @@ public class DocumentGraphMapper {
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
-            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/PropertiesMapper.java

@@ -1,16 +1,18 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.document.data.mapper;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
 import lombok.experimental.UtilityClass;
 
 @UtilityClass

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document;
+package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import static java.lang.String.format;
 
@@ -6,21 +6,20 @@ import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
 
 @Setter
 @EqualsAndHashCode
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
-public class TextRange implements Comparable<TextRange> {
+public class Boundary implements Comparable<Boundary> {
 
     private int start;
     private int end;
 
 
-    public TextRange(int start, int end) {
+    public Boundary(int start, int end) {
 
         if (start > end) {
             throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
@@ -48,15 +47,15 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
-    public boolean contains(TextRange textRange) {
+    public boolean contains(Boundary boundary) {
 
-        return start <= textRange.start() && textRange.end() <= end;
+        return start <= boundary.start() && boundary.end() <= end;
     }
 
 
-    public boolean containedBy(TextRange textRange) {
+    public boolean containedBy(Boundary boundary) {
 
-        return textRange.contains(this);
+        return boundary.contains(this);
     }
 
 
@@ -84,18 +83,18 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
-    public boolean intersects(TextRange textRange) {
+    public boolean intersects(Boundary boundary) {
 
-        return textRange.start() < this.end && this.start < textRange.end();
+        return boundary.start() < this.end && this.start < boundary.end();
     }
 
 
-    public List<TextRange> split(List<Integer> splitIndices) {
+    public List<Boundary> split(List<Integer> splitIndices) {
 
         if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
             throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
         }
-        List<TextRange> splitBoundaries = new LinkedList<>();
+        List<Boundary> splitBoundaries = new LinkedList<>();
         int previousIndex = start;
         for (int splitIndex : splitIndices) {
 
@@ -103,19 +102,19 @@ public class TextRange implements Comparable<TextRange> {
             if (splitIndex == previousIndex) {
                 continue;
             }
-            splitBoundaries.add(new TextRange(previousIndex, splitIndex));
+            splitBoundaries.add(new Boundary(previousIndex, splitIndex));
             previousIndex = splitIndex;
         }
-        splitBoundaries.add(new TextRange(previousIndex, end));
+        splitBoundaries.add(new Boundary(previousIndex, end));
         return splitBoundaries;
     }
 
 
-    public static TextRange merge(Collection<TextRange> boundaries) {
+    public static Boundary merge(Collection<Boundary> boundaries) {
 
-        int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new);
-        int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new);
-        return new TextRange(minStart, maxEnd);
+        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
+        int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
+        return new Boundary(minStart, maxEnd);
     }
 
 
@@ -127,12 +126,12 @@ public class TextRange implements Comparable<TextRange> {
 
 
     @Override
-    public int compareTo(TextRange textRange) {
+    public int compareTo(Boundary boundary) {
 
-        if (end < textRange.end() && start < textRange.start()) {
+        if (end < boundary.end() && start < boundary.start()) {
             return -1;
         }
-        if (start > textRange.start() && end > textRange.end()) {
+        if (start > boundary.start() && end > boundary.end()) {
             return 1;
         }
 
@@ -146,7 +145,7 @@ public class TextRange implements Comparable<TextRange> {
      * @param textBlock TextBlock to check whitespaces against
      * @return trimmed boundary
      */
-    public TextRange trim(TextBlock textBlock) {
+    public Boundary trim(TextBlock textBlock) {
 
         if (this.length() == 0) {
             return this;
@@ -161,7 +160,7 @@ public class TextRange implements Comparable<TextRange> {
             trimmedEnd--;
         }
 
-        return new TextRange(trimmedStart, Math.max(trimmedEnd, trimmedStart));
+        return new Boundary(trimmedStart, Math.max(trimmedEnd, trimmedStart));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ConsecutiveBoundaryCollector.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document;
+package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -11,17 +11,17 @@ import java.util.stream.Collector;
 
 import com.google.common.base.Functions;
 
-public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<TextRange>, List<TextRange>> {
+public class ConsecutiveBoundaryCollector implements Collector<Boundary, List<Boundary>, List<Boundary>> {
 
     @Override
-    public Supplier<List<TextRange>> supplier() {
+    public Supplier<List<Boundary>> supplier() {
 
         return LinkedList::new;
     }
 
 
     @Override
-    public BiConsumer<List<TextRange>, TextRange> accumulator() {
+    public BiConsumer<List<Boundary>, Boundary> accumulator() {
 
         return (existingList, boundary) -> {
             if (existingList.isEmpty()) {
@@ -29,14 +29,14 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
                 return;
             }
 
-            TextRange prevTextRange = existingList.get(existingList.size() - 1);
-            if (prevTextRange.end() > boundary.start()) {
-                throw new IllegalArgumentException(String.format("Can't concatenate %s and %s. Boundaries must be ordered!", prevTextRange, boundary));
+            Boundary prevBoundary = existingList.get(existingList.size() - 1);
+            if (prevBoundary.end() > boundary.start()) {
+                throw new IllegalArgumentException(String.format("Can't concatenate %s and %s. Boundaries must be ordered!", prevBoundary, boundary));
             }
 
-            if (prevTextRange.end() == boundary.start()) {
+            if (prevBoundary.end() == boundary.start()) {
                 existingList.remove(existingList.size() - 1);
-                existingList.add(TextRange.merge(List.of(prevTextRange, boundary)));
+                existingList.add(Boundary.merge(List.of(prevBoundary, boundary)));
             } else {
                 existingList.add(boundary);
             }
@@ -45,7 +45,7 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
 
 
     @Override
-    public BinaryOperator<List<TextRange>> combiner() {
+    public BinaryOperator<List<Boundary>> combiner() {
 
         return (list1, list2) -> {
             list1.addAll(list2);
@@ -55,7 +55,7 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
 
 
     @Override
-    public Function<List<TextRange>, List<TextRange>> finisher() {
+    public Function<List<Boundary>, List<Boundary>> finisher() {
 
         return Functions.identity();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTree.java

@@ -1,21 +1,20 @@
-package com.iqser.red.service.redaction.v1.server.model.document;
+package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import static java.lang.String.format;
 
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.GenericSemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -136,48 +135,6 @@ public class DocumentTree {
     }
 
 
-    public Optional<SemanticNode> getNextSibling(List<Integer> treeId) {
-
-        var siblingTreeId = getNextSiblingId(treeId);
-        if (!entryExists(siblingTreeId)) {
-            return Optional.empty();
-        }
-        return Optional.of(getEntryById(siblingTreeId).getNode());
-    }
-
-
-    public List<Integer> getNextSiblingId(List<Integer> treeId) {
-
-        List<Integer> siblingTreeId = new LinkedList<>();
-        for (int i = 0; i < treeId.size() - 1; i++) {
-            siblingTreeId.add(treeId.get(i));
-        }
-        siblingTreeId.add(treeId.get(treeId.size() - 1) + 1);
-        return siblingTreeId;
-    }
-
-
-    public Optional<SemanticNode> getPreviousSibling(List<Integer> treeId) {
-
-        var siblingTreeId = getPreviousSiblingId(treeId);
-        if (!entryExists(siblingTreeId)) {
-            return Optional.empty();
-        }
-        return Optional.of(getEntryById(siblingTreeId).getNode());
-    }
-
-
-    public List<Integer> getPreviousSiblingId(List<Integer> treeId) {
-
-        List<Integer> siblingTreeId = new LinkedList<>();
-        for (int i = 0; i < treeId.size() - 1; i++) {
-            siblingTreeId.add(treeId.get(i));
-        }
-        siblingTreeId.add(treeId.get(treeId.size() - 1) - 1);
-        return siblingTreeId;
-    }
-
-
     public Entry getEntryById(List<Integer> treeId) {
 
         if (treeId.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityType.java

@@ -1,8 +1,7 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
 
 public enum EntityType {
     ENTITY,
-    HINT,
     RECOMMENDATION,
     FALSE_POSITIVE,
     FALSE_RECOMMENDATION

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRule.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@AllArgsConstructor
+@EqualsAndHashCode
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public final class MatchedRule implements Comparable<MatchedRule> {
+
+    @Builder.Default
+    RuleIdentifier ruleIdentifier = RuleIdentifier.empty();
+    @Builder.Default
+    String reason = "";
+    @Builder.Default
+    String legalBasis = "";
+    boolean applied;
+    boolean writeValueWithLineBreaks;
+    @Builder.Default
+    Set<RedactionEntity> references = Collections.emptySet();
+
+
+    public static MatchedRule empty() {
+
+        return MatchedRule.builder().build();
+    }
+
+
+    @Override
+    public int compareTo(MatchedRule matchedRule) {
+
+        RuleIdentifier otherRuleIdentifier = matchedRule.getRuleIdentifier();
+        if (!Objects.equals(ruleIdentifier.type(), otherRuleIdentifier.type())) {
+            if (Objects.equals(otherRuleIdentifier.type(), "MAN")) {
+                return 1;
+            }
+            if (Objects.equals(ruleIdentifier.type(), "MAN")) {
+                return -1;
+            }
+            if (Objects.equals(otherRuleIdentifier.type(), "X")) {
+                return 1;
+            }
+            if (Objects.equals(ruleIdentifier.type(), "X")) {
+                return -1;
+            }
+        }
+        if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) {
+            return otherRuleIdentifier.unit() - ruleIdentifier.unit();
+        }
+        return otherRuleIdentifier.id() - ruleIdentifier.id();
+    }
+
+
+    @Override
+    public String toString() {
+
+        return "MatchedRule[" + "ruleIdentifier=" + ruleIdentifier + ", " + "reason=" + reason + ", " + "legalBasis=" + legalBasis + ", " + "applied=" + applied + ", " + "writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", " + "references=" + references + ']';
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRuleHolder.java

@@ -1,111 +1,64 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
 
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.PriorityQueue;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
-
 import lombok.NonNull;
 
-public interface IEntity {
+public interface MatchedRuleHolder {
 
     PriorityQueue<MatchedRule> getMatchedRuleList();
 
 
-    ManualChangeOverwrite getManualOverwrite();
+    boolean isIgnored();
 
 
-    String getValue();
+    boolean isRemoved();
 
 
-    TextRange getTextRange();
+    void setIgnored(boolean ignored);
 
 
-    String type();
+    void setRemoved(boolean ignored);
 
 
-    default int length() {
+    default boolean isApplied() {
 
-        return value().length();
+        return getMatchedRule().isApplied();
     }
 
 
-    default String value() {
-
-        return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
-    }
-
-
-    // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
-    default boolean applied() {
-
-        return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied());
-    }
-
-
-    default boolean skipped() {
-
-        return !applied();
-    }
-
-
-    default boolean ignored() {
-
-        return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored());
-    }
-
-
-    default boolean removed() {
-
-        return getManualOverwrite().getRemoved().orElse(getMatchedRule().isRemoved());
-    }
-
-
-    default boolean resized() {
-
-        return getManualOverwrite().getResized().orElse(false);
-    }
-
-
-    default boolean active() {
-
-        return !(removed() || ignored());
-    }
-
-
-    default boolean hasManualChanges() {
-
-        return !getManualOverwrite().getManualChangeLog().isEmpty();
-    }
-
-
-    default Set<TextEntity> references() {
+    default Set<RedactionEntity> getReferences() {
 
         return getMatchedRule().getReferences();
     }
 
 
-    default void redact(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
+    default boolean isActive() {
+
+        return !(isRemoved() || isIgnored());
+    }
+
+
+    default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
 
         if (legalBasis.isBlank() || legalBasis.isEmpty()) {
             throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
         }
-        apply(ruleIdentifier, reason, legalBasis);
-    }
-
-
-    default void apply(@NonNull String ruleIdentifier, String reason, String legalBasis) {
-
         addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
     }
 
 
-    default void apply(@NonNull String ruleIdentifier, String reason) {
+    default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) {
 
-        apply(ruleIdentifier, reason, "n-a");
+        addMatchedRule(MatchedRule.builder()
+                .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
+                .reason(reason)
+                .legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis))
+                .applied(true)
+                .build());
     }
 
 
@@ -117,13 +70,27 @@ public interface IEntity {
 
     default void remove(String ruleIdentifier, String reason) {
 
-        addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).removed(true).build());
+        addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
+        setRemoved(true);
     }
 
 
     default void ignore(String ruleIdentifier, String reason) {
 
-        addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).ignored(true).build());
+        addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
+        setIgnored(true);
+    }
+
+
+    private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) {
+
+        if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) {
+            if (getMatchedRule() == null || !getMatchedRule().isApplied()) {
+                return "n-a";
+            }
+            return getMatchedRule().getLegalBasis();
+        }
+        return legalBasis;
     }
 
 
@@ -142,7 +109,7 @@ public interface IEntity {
     }
 
 
-    default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection<TextEntity> references) {
+    default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection<RedactionEntity> references) {
 
         if (legalBasis.isBlank() || legalBasis.isEmpty()) {
             throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
@@ -157,7 +124,7 @@ public interface IEntity {
     }
 
 
-    default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<TextEntity> references) {
+    default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<RedactionEntity> references) {
 
         getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build());
     }
@@ -171,9 +138,6 @@ public interface IEntity {
 
     default void addMatchedRules(Collection<MatchedRule> matchedRules) {
 
-        if (getMatchedRuleList().equals(matchedRules)) {
-            return;
-        }
         getMatchedRuleList().addAll(matchedRules);
     }
 
@@ -192,22 +156,4 @@ public interface IEntity {
         return getMatchedRuleList().peek();
     }
 
-
-    default String buildReasonWithManualChangeDescriptions() {
-
-        if (getManualOverwrite().getDescriptions().isEmpty()) {
-            return getMatchedRule().getReason();
-        }
-        if (getMatchedRule().getReason().isEmpty()) {
-            return String.join(", ", getManualOverwrite().getDescriptions());
-        }
-        return getMatchedRule().getReason() + ", " + String.join(", ", getManualOverwrite().getDescriptions());
-    }
-
-
-    default String legalBasis() {
-
-        return getManualOverwrite().getLegalBasis().orElse(getMatchedRule().getLegalBasis());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RedactionEntity.java

@@ -0,0 +1,205 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
+public class RedactionEntity implements MatchedRuleHolder {
+
+    // initial values
+    @EqualsAndHashCode.Include
+    final Boundary boundary;
+    @EqualsAndHashCode.Include
+    final String type;
+    @EqualsAndHashCode.Include
+    final EntityType entityType;
+
+    // empty defaults
+    boolean removed;
+    boolean ignored;
+
+    boolean resized;
+    boolean skipRemoveEntitiesContainedInLarger;
+    boolean dictionaryEntry;
+    boolean dossierDictionaryEntry;
+    @Builder.Default
+    Set<Engine> engines = new HashSet<>();
+
+    @Builder.Default
+    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
+
+    // inferred on graph insertion
+    String value;
+    String textBefore;
+    String textAfter;
+    @Builder.Default
+    Set<Page> pages = new HashSet<>();
+    List<RedactionPosition> redactionPositionsPerPage;
+    @Builder.Default
+    List<SemanticNode> intersectingNodes = new LinkedList<>();
+    SemanticNode deepestFullyContainingNode;
+
+
+    public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) {
+
+        return RedactionEntity.builder().type(type).entityType(entityType).boundary(boundary).build();
+    }
+
+
+    public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
+
+        return intersectingNodes.stream().anyMatch(clazz::isInstance);
+    }
+
+
+    public boolean occursInNode(SemanticNode semanticNode) {
+
+        return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode));
+    }
+
+
+    public boolean isType(String type) {
+
+        return this.type.equals(type);
+    }
+
+
+    public boolean isAnyType(List<String> types) {
+
+        return types.contains(type);
+    }
+
+
+    public void addIntersectingNode(SemanticNode containingNode) {
+
+        intersectingNodes.add(containingNode);
+    }
+
+
+    public String getValueWithLineBreaks() {
+
+        return getDeepestFullyContainingNode().getTextBlock().subSequenceWithLineBreaks(getBoundary());
+    }
+
+
+    public void removeFromGraph() {
+
+        intersectingNodes.forEach(node -> node.getEntities().remove(this));
+        pages.forEach(page -> page.getEntities().remove(this));
+        intersectingNodes = new LinkedList<>();
+        deepestFullyContainingNode = null;
+        pages = new HashSet<>();
+        removed = true;
+    }
+
+
+    public List<RedactionPosition> getRedactionPositionsPerPage() {
+
+        if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {
+            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(boundary);
+
+            Page firstPage = rectanglesPerLinePerPage.keySet()
+                    .stream()
+                    .min(Comparator.comparingInt(Page::getNumber))
+                    .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
+
+            String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
+            redactionPositionsPerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildRedactionPosition(firstPage, id, entry)).toList();
+        }
+        return redactionPositionsPerPage;
+    }
+
+
+    private static RedactionPosition buildRedactionPosition(Page firstPage, String id, Map.Entry<Page, List<Rectangle2D>> entry) {
+
+        if (entry.getKey().equals(firstPage)) {
+            return new RedactionPosition(id, entry.getKey(), entry.getValue());
+        } else {
+            return new RedactionPosition(id + "-" + entry.getKey().getNumber(), entry.getKey(), entry.getValue());
+        }
+    }
+
+
+    public boolean containedBy(RedactionEntity redactionEntity) {
+
+        return this.boundary.containedBy(redactionEntity.getBoundary());
+    }
+
+
+    public boolean contains(RedactionEntity redactionEntity) {
+
+        return this.boundary.contains(redactionEntity.getBoundary());
+    }
+
+
+    public boolean intersects(RedactionEntity redactionEntity) {
+
+        return this.boundary.intersects(redactionEntity.getBoundary());
+    }
+
+
+    public void addEngine(Engine engine) {
+
+        engines.add(engine);
+    }
+
+
+    public void addEngines(Set<Engine> engines) {
+
+        this.engines.addAll(engines);
+    }
+
+
+    public boolean matchesAnnotationId(String manualRedactionId) {
+
+        return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("Entity[\"");
+        sb.append(value);
+        sb.append("\", ");
+        sb.append(boundary);
+        sb.append(", pages[");
+        pages.forEach(page -> {
+            sb.append(page.getNumber());
+            sb.append(", ");
+        });
+        sb.delete(sb.length() - 2, sb.length());
+        sb.append("], type = \"");
+        sb.append(type);
+        sb.append("\", EntityType.");
+        sb.append(entityType);
+        sb.append("]");
+        return sb.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RedactionPosition.java

@@ -1,9 +1,9 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -13,13 +13,12 @@ import lombok.experimental.FieldDefaults;
 @Data
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class PositionOnPage {
+public class RedactionPosition {
 
-    // Each entry in this list corresponds to an entry in the redaction log, this means:
-    // A single entity might be represented by multiple redaction log entries
-    // This is due to the RedactionLog only being able to handle a single page per entry.
     final String id;
     Page page;
+    // Each entry in this list corresponds to an entry in the redaction log, this means:
+    // An entity might be represented by multiple redaction log entries
     List<Rectangle2D> rectanglePerLine;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RuleIdentifier.java

@@ -0,0 +1,41 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+
+import java.util.Objects;
+
+public record RuleIdentifier(String type, Integer unit, Integer id) {
+
+    public static RuleIdentifier fromString(String identifier) {
+
+        String[] values = identifier.split("\\.");
+        if (values.length != 3) {
+            throw new IllegalArgumentException("Illegal rule identifier provided: " + identifier);
+        }
+        String type = values[0];
+        Integer group = Integer.parseInt(values[1]);
+        Integer id = Integer.parseInt(values[2]);
+        return new RuleIdentifier(type, group, id);
+    }
+
+
+    public static RuleIdentifier empty() {
+
+        return new RuleIdentifier("", null, null);
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(type());
+        if (Objects.nonNull(unit()) && Objects.nonNull(id())) {
+            sb.append(".").append(unit()).append(".").append(id());
+        } else if (Objects.nonNull(id())) {
+            sb.append(".*.").append(id());
+        } else if (Objects.nonNull(unit())) {
+            sb.append(".").append(unit()).append(".*");
+        }
+        return sb.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Document.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -10,17 +10,15 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
@@ -29,20 +27,14 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Document implements GenericSemanticNode {
 
-    @EqualsAndHashCode.Include
-    List<Integer> treeId = Collections.emptyList();
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
     Set<Page> pages;
     DocumentTree documentTree;
     Integer numberOfPages;
     TextBlock textBlock;
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
+    Set<RedactionEntity> entities = new HashSet<>();
     @Builder.Default
     static final SectionIdentifier sectionIdentifier = SectionIdentifier.document();
 
@@ -75,6 +67,13 @@ public class Document implements GenericSemanticNode {
     }
 
 
+    @Override
+    public List<Integer> getTreeId() {
+
+        return Collections.emptyList();
+    }
+
+
     @Override
     public void setTreeId(List<Integer> tocId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Footer.java

@@ -1,15 +1,12 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,24 +21,20 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Footer implements GenericSemanticNode {
 
-    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
     @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
-
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override
@@ -78,14 +71,4 @@ public class Footer implements GenericSemanticNode {
         return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/GenericSemanticNode.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 public interface GenericSemanticNode extends SemanticNode {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Header.java

@@ -1,15 +1,12 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,24 +21,20 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Header implements GenericSemanticNode {
 
-    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
     @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
-
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override
@@ -78,14 +71,4 @@ public class Header implements GenericSemanticNode {
         return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Headline.java

@@ -1,16 +1,13 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -25,22 +22,18 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
     SectionIdentifier sectionIdentifier;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override
@@ -78,16 +71,6 @@ public class Headline implements GenericSemanticNode {
     }
 
 
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
-
     @Override
     public SectionIdentifier getSectionIdentifier() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Image.java

@@ -0,0 +1,94 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRuleHolder;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class Image implements GenericSemanticNode, MatchedRuleHolder {
+
+    List<Integer> treeId;
+    String id;
+
+    ImageType imageType;
+    boolean transparent;
+    Rectangle2D position;
+
+    boolean removed;
+    boolean ignored;
+
+    @Builder.Default
+    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
+
+    @EqualsAndHashCode.Exclude
+    Page page;
+
+    @EqualsAndHashCode.Exclude
+    DocumentTree documentTree;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
+
+
+    @Override
+    public NodeType getType() {
+
+        return NodeType.IMAGE;
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+    }
+
+
+    @Override
+    public Set<Page> getPages() {
+
+        return Collections.singleton(page);
+    }
+
+
+    @Override
+    public String toString() {
+
+        return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
+    }
+
+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
+        bBoxPerPage.put(page, position);
+        return bBoxPerPage;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageType.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.util.Locale;
 
@@ -6,22 +6,10 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER {
-        @Override
-        public String toString() {
-
-            return "image";
-        }
-    },
+    OTHER,
     OCR;
 
 
-    public String toString() {
-
-        return name().toLowerCase(Locale.ENGLISH);
-    }
-
-
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.util.Locale;
 
@@ -16,6 +16,6 @@ public enum NodeType {
 
     public String toString() {
 
-        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH);
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Page.java

@@ -1,12 +1,13 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -23,23 +24,26 @@ import lombok.experimental.FieldDefaults;
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Page {
 
-    @EqualsAndHashCode.Include
     Integer number;
     Integer height;
     Integer width;
     Integer rotation;
 
+    @EqualsAndHashCode.Exclude
     List<SemanticNode> mainBody;
+    @EqualsAndHashCode.Exclude
     Header header;
+    @EqualsAndHashCode.Exclude
     Footer footer;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<Image> images = new HashSet<>();
 
 
@@ -54,4 +58,19 @@ public class Page {
 
         return String.valueOf(number);
     }
+
+
+    @Override
+    public int hashCode() {
+
+        return number;
+    }
+
+
+    @Override
+    public boolean equals(Object o) {
+
+        return o instanceof Page && o.hashCode() == this.hashCode();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Paragraph.java

@@ -1,15 +1,12 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -22,22 +19,17 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override
@@ -67,14 +59,4 @@ public class Paragraph implements GenericSemanticNode {
         return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Section.java

@@ -1,23 +1,19 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,22 +22,17 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
 
     TextBlock textBlock;
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
-
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override
@@ -51,6 +42,11 @@ public class Section implements GenericSemanticNode {
     }
 
 
+    /**
+     * Determines whether this Section has any tables.
+     *
+     * @return {@code true} if there are tables, {@code false} otherwise
+     */
     public boolean hasTables() {
 
         return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
@@ -101,14 +97,4 @@ public class Section implements GenericSemanticNode {
         return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionIdentifier.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.util.Collections;
 import java.util.LinkedList;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import static java.lang.String.format;
 
@@ -9,20 +9,17 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
-import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility;
 
 public interface SemanticNode {
 
@@ -46,11 +43,11 @@ public interface SemanticNode {
 
     /**
      * Any Node maintains its own Set of Entities.
-     * This Set contains all Entities whose TextRange intersects the TextRange of this node.
+     * This Set contains all Entities whose boundary intersects the boundary of this node.
      *
      * @return Set of all Entities associated with this Node
      */
-    Set<TextEntity> getEntities();
+    Set<RedactionEntity> getEntities();
 
 
     /**
@@ -63,7 +60,6 @@ public interface SemanticNode {
         return getTextBlock().getPages();
     }
 
-
     /**
      * Finds the first page associated with this Node.
      *
@@ -76,25 +72,19 @@ public interface SemanticNode {
 
 
     /**
-     * Each AtomicTextBlock is assigned a page, so to get the pages for this TextRange, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
+     * Each AtomicTextBlock is assigned a page, so to get the pages for this boundary, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
      *
      * @return Set of PageNodes this node appears on.
      */
-    default Set<Page> getPages(TextRange textRange) {
+    default Set<Page> getPages(Boundary boundary) {
 
-        if (!getTextRange().intersects(textRange)) {
-            throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
+        if (!getBoundary().contains(boundary)) {
+            throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", boundary, getBoundary()));
         }
-        return getTextBlock().getPages(textRange);
+        return getTextBlock().getPages(boundary);
     }
 
 
-    /**
-     * Checks if the given page number exists in the list of pages.
-     *
-     * @param pageNumber the page number to be checked
-     * @return true if the page number exists, otherwise false
-     */
     default boolean onPage(int pageNumber) {
 
         return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
@@ -180,30 +170,6 @@ public interface SemanticNode {
     }
 
 
-    /**
-     * Returns the next sibling node of this SemanticNode in the document tree, if any.
-     * If there is no next sibling node, an empty Optional is returned.
-     *
-     * @return Optional containing the next sibling node, or empty if there is none
-     */
-    default Optional<SemanticNode> getNextSibling() {
-
-        return getDocumentTree().getNextSibling(getTreeId());
-    }
-
-
-    /**
-     * Returns the previous sibling node of this SemanticNode in the document tree, if any.
-     * If there is no previous sibling node, an empty Optional is returned.
-     *
-     * @return Optional containing the previous sibling node, or empty if there is none
-     */
-    default Optional<SemanticNode> getPreviousSibling() {
-
-        return getDocumentTree().getPreviousSibling(getTreeId());
-    }
-
-
     /**
      * Leaf means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
      * Currently only Sections, Images, and Tables are not leaves.
@@ -249,7 +215,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
+        return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
     }
 
 
@@ -258,11 +224,11 @@ public interface SemanticNode {
      * Ignores Entity with ignored == true or removed == true.
      *
      * @param types an array of strings representing the types of entities to check for
-     * @return true, if this SemanticNode has at least one Entity of the provided types
+     * @return true, if this SemanticNode has at least one Entity of any of the provided types
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
+        return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
     }
 
 
@@ -275,7 +241,11 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
+        return getEntities().stream()
+                .filter(RedactionEntity::isActive)
+                .map(RedactionEntity::getType)
+                .collect(Collectors.toUnmodifiableSet())
+                .containsAll(Arrays.stream(types).toList());
     }
 
 
@@ -286,9 +256,9 @@ public interface SemanticNode {
      * @param type string representing the type of entities to return
      * @return List of RedactionEntities of any the type
      */
-    default List<TextEntity> getEntitiesOfType(String type) {
+    default List<RedactionEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
+        return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
     }
 
 
@@ -299,9 +269,9 @@ public interface SemanticNode {
      * @param types A list of strings representing the types of entities to return
      * @return List of RedactionEntities of any provided type
      */
-    default List<TextEntity> getEntitiesOfType(List<String> types) {
+    default List<RedactionEntity> getEntitiesOfType(List<String> types) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
+        return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
     }
 
 
@@ -312,9 +282,9 @@ public interface SemanticNode {
      * @param types A list of strings representing the types of entities to return
      * @return List of RedactionEntities that match any of the provided types
      */
-    default List<TextEntity> getEntitiesOfType(String... types) {
+    default List<RedactionEntity> getEntitiesOfType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
+        return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
     }
 
 
@@ -357,13 +327,6 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
-    Set<LayoutEngine> getEngines();
-
-    default void addEngine(LayoutEngine engine) {
-        getEngines().add(engine);
-    }
-
-
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -437,78 +400,6 @@ public interface SemanticNode {
     }
 
 
-    /**
-     * Checks whether this SemanticNode contains exactly the provided String as a word.
-     *
-     * @param word - String which the TextBlock might contain
-     * @return true, if this node's TextBlock contains string
-     */
-    default boolean containsWord(String word) {
-
-        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
-     *
-     * @param word - String which the TextBlock might contain
-     * @return true, if this node's TextBlock contains string
-     */
-    default boolean containsWordIgnoreCase(String word) {
-
-        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains any of the provided Strings as a word.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains any of the provided strings
-     */
-    default boolean containsAnyWord(String... words) {
-
-        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains any of the provided strings
-     */
-    default boolean containsAnyWordIgnoreCase(String... words) {
-
-        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains all the provided Strings as word.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains all the provided strings
-     */
-    default boolean containsAllWords(String... words) {
-
-        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains all the provided strings
-     */
-    default boolean containsAllWordsIgnoreCase(String... words) {
-
-        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
-    }
-
-
     /**
      * Checks whether this SemanticNode matches the provided regex pattern.
      *
@@ -534,13 +425,12 @@ public interface SemanticNode {
 
 
     /**
-     * Checks whether this SemanticNode intersects the provided rectangle.
-     *
-     * @param x          the lower left corner X value
-     * @param y          the lower left corner Y value
-     * @param w          width
-     * @param h          height
-     * @param pageNumber the pageNumber of the rectangle
+     * Checks whether this SemanticNode intersects the provided rectangle
+     * @param x the lower left corner X value
+     * @param y the lower left corner Y value
+     * @param w width
+     * @param h height
+     * @param pageNumber the pagenumber of the rectangle
      * @return true if intersects, false otherwise
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
@@ -550,21 +440,22 @@ public interface SemanticNode {
 
 
     /**
-     * This function is used during insertion of EntityNodes into the graph, it checks if the TextRange of the RedactionEntity intersects or even contains the RedactionEntity.
+     * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
      * It sets the fields accordingly and recursively calls this function on all its children.
      *
-     * @param textEntity RedactionEntity, which is being inserted into the graph
+     * @param redactionEntity RedactionEntity, which is being inserted into the graph
      */
-    default void addThisToEntityIfIntersects(TextEntity textEntity) {
+    default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) {
 
         TextBlock textBlock = getTextBlock();
-        if (textBlock.getTextRange().intersects(textEntity.getTextRange())) {
-            if (textBlock.containsTextRange(textEntity.getTextRange())) {
-                textEntity.setDeepestFullyContainingNode(this);
+        if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
+            if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
+                redactionEntity.setDeepestFullyContainingNode(this);
             }
-            textEntity.addIntersectingNode(this);
-            streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
-                    .forEach(node -> node.addThisToEntityIfIntersects(textEntity));
+
+            redactionEntity.addIntersectingNode(this);
+            streamChildren().filter(semanticNode -> semanticNode.getBoundary().intersects(redactionEntity.getBoundary()))
+                    .forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
         }
     }
 
@@ -614,13 +505,13 @@ public interface SemanticNode {
 
 
     /**
-     * The TextRange is the start and end string offsets in the reading order of the document.
+     * The Boundary is the start and end string offsets in the reading order of the document.
      *
-     * @return TextRange of this Node's TextBlock
+     * @return Boundary of this Node's TextBlock
      */
-    default TextRange getTextRange() {
+    default Boundary getBoundary() {
 
-        return getTextBlock().getTextRange();
+        return getTextBlock().getBoundary();
     }
 
 
@@ -631,28 +522,7 @@ public interface SemanticNode {
      */
     default int length() {
 
-        return getTextRange().length();
-    }
-
-
-    /**
-     * For a given TextRange this function returns a List of rectangle around the text in the range.
-     * These Rectangles are split either by a new line or by a large gap in the current line.
-     * This is mainly used to find the positions of TextEntities
-     *
-     * @param textRange A TextRange to calculate the positions for.
-     * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
-     */
-    default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
-
-        if (isLeaf()) {
-            return getTextBlock().getPositionsPerPage(textRange);
-        }
-        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
-        if (containingChildNode.isEmpty()) {
-            return getTextBlock().getPositionsPerPage(textRange);
-        }
-        return containingChildNode.get().getPositionsPerPage(textRange);
+        return getBoundary().length();
     }
 
 
@@ -664,11 +534,12 @@ public interface SemanticNode {
      */
     default Map<Page, Rectangle2D> getBBox() {
 
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
         if (isLeaf()) {
-            return getBBoxFromLeafTextBlock();
+            return getBBoxFromLeafTextBlock(bBoxPerPage);
         }
 
-        return getBBoxFromChildren();
+        return getBBoxFromChildren(bBoxPerPage);
     }
 
 
@@ -691,49 +562,28 @@ public interface SemanticNode {
 
     /**
      * TODO: this produces unwanted results for sections spanning multiple columns.
-     * Computes the Union of the bounding boxes of all children recursively.
      *
+     * @param bBoxPerPage initial empty BoundingBox
      * @return The union of the BoundingBoxes of all children
      */
-    private Map<Page, Rectangle2D> getBBoxFromChildren() {
+    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
 
-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
-        for (Page page : pages) {
-            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
-                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
-                    .map(childBboxPerPage -> childBboxPerPage.get(page))
-                    .collect(RectangleTransformations.collectBBox());
-            bBoxPerPage.put(page, bBoxOnPage);
-        }
-        return bBoxPerPage;
+        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
+            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
+            return map2;
+        }).orElse(bBoxPerPage);
     }
 
 
     /**
+     * @param bBoxPerPage initial empty BoundingBox
      * @return The union of all BoundingBoxes of the TextBlock of this node
      */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
 
-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
         Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
+        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs)));
         return bBoxPerPage;
     }
 
-
-    /**
-     * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
-     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
-     *
-     * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
-     * @see NodeVisitor
-     */
-    default void accept(NodeVisitor visitor) {
-
-        visitor.visit(this);
-        streamChildren().forEach(childNode -> childNode.accept(visitor));
-    }
-
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Table.java

@@ -1,23 +1,19 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -30,12 +26,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
 
@@ -45,19 +37,8 @@ public class Table implements SemanticNode {
     TextBlock textBlock;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
-
-    Map<Page, Rectangle2D> bBoxCache;
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = SemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     /**
@@ -66,9 +47,10 @@ public class Table implements SemanticNode {
      * @param strings Strings to check whether a row contains them
      * @return Stream of all entities in this table, that appear in a row, which contains any of the provided strings
      */
-    public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
 
-        return IntStream.range(0, numberOfRows).boxed()
+        return IntStream.range(0, numberOfRows)
+                .boxed()
                 .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
@@ -85,11 +67,8 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock)
-                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream()
-                .map(String::toLowerCase)
-                .allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
     }
 
 
@@ -100,15 +79,11 @@ public class Table implements SemanticNode {
      * @param value  the string which the table cell should contain
      * @return a stream of all entities, which appear in a row where at least one cell has the provided header and the provided value.
      */
-    public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
-                .map(TableCell::getCol)
-                .toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
         return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -119,15 +94,11 @@ public class Table implements SemanticNode {
      * @param values the strings which the table cell should contain
      * @return a stream of all entities, which appear in a row where at least one cell has the provided header and any provided value.
      */
-    public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
-                .map(TableCell::getCol)
-                .toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
         return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -138,35 +109,18 @@ public class Table implements SemanticNode {
      * @param types type strings to check whether a row contains an entity like them
      * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
      */
-    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                        .anyMatch(types::contains))
-                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
-    }
+        List<Integer> rowsWithEntityOfType = getEntities().stream()
+                .filter(RedactionEntity::isActive)
+                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
+                .map(RedactionEntity::getIntersectingNodes)
+                .filter(node -> node instanceof TableCell)
+                .map(node -> (TableCell) node)
+                .map(TableCell::getRow)
+                .toList();
 
-
-    /**
-     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
-     * Ignores Entity with ignored == true or removed == true.
-     *
-     * @param types type strings to check whether a row contains an entity like them
-     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
-     */
-    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
-
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> {
-                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                            .collect(Collectors.toSet());
-                    return entityTypes.containsAll(types);
-                })
-                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -177,45 +131,20 @@ public class Table implements SemanticNode {
      * @param types type strings to check whether a row contains an entity like them
      * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
      */
-    public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                        .noneMatch(types::contains))
+        return IntStream.range(0, numberOfRows)
+                .boxed()
+                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
+                        .flatMap(Collection::stream)
+                        .filter(RedactionEntity::isActive)
+                        .noneMatch(entity -> types.contains(entity.getType())))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
     }
 
 
-    /**
-     * Streams all Entities in the given row.
-     *
-     * @param rowNumber the row number to look for
-     * @return stream of TextEntities occurring in row
-     */
-    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
-
-        return streamRow(rowNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
-    }
-
-
-    /**
-     * Streams all Entities in the given col.
-     *
-     * @param colNumber the column number to look for
-     * @return stream of TextEntities occurring in row
-     */
-    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
-
-        return streamCol(colNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
-    }
-
-
     /**
      * Returns a TableCell at the provided row and column location.
      *
@@ -229,8 +158,7 @@ public class Table implements SemanticNode {
             throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
         }
         int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren()
-                .get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
     }
 
 
@@ -245,18 +173,6 @@ public class Table implements SemanticNode {
     }
 
 
-    /**
-     * Streams all TableCells that contain at least one entity of a given type in this Table row-wise.
-     *
-     * @param type The type of the entity
-     * @return Stream of filtered TableCells
-     */
-    public Stream<TableCell> streamTableCellsWhichContainType(String type) {
-
-        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
-    }
-
-
     /**
      * Streams all TableCells in this Table which have the provided header row-wise.
      *
@@ -279,8 +195,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamCol(int col) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
     }
 
 
@@ -292,8 +207,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamRow(int row) {
 
-        return IntStream.range(0, numberOfCols).boxed()
-                .map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
     }
 
 
@@ -317,8 +231,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamHeadersForCell(int row, int col) {
 
-        return Stream.concat(streamRow(row), streamCol(col))
-                .filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
     }
 
 
@@ -334,22 +247,6 @@ public class Table implements SemanticNode {
     }
 
 
-    /**
-     * Streams all Headers and checks if any equal the provided string.
-     *
-     * @param header string to check the headers for
-     * @return true, if at least one header equals the provided string
-     */
-    public boolean hasHeaderIgnoreCase(String header) {
-
-        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock()
-                .getSearchText()
-                .strip()
-                .toLowerCase(Locale.ENGLISH)
-                .equals(header.toLowerCase(Locale.ENGLISH)));
-    }
-
-
     /**
      * Checks if this table has a column with the provided header and any of the table cells in that column contain the provided value.
      *
@@ -370,7 +267,7 @@ public class Table implements SemanticNode {
      * @param values List of strings to check cells with provided header
      * @return true, if this table has a column with the provided header and any of the table cells in that column contains any of the provided values.
      */
-    public boolean hasRowWithHeaderAndAnyValue(String header, List<String> values) {
+    public boolean hasRowWithHeaderAndAnyValue(String header, String... values) {
 
         return streamTableCellsWithHeader(header).anyMatch(tableCellNode -> tableCellNode.containsAnyString(values));
     }
@@ -380,13 +277,13 @@ public class Table implements SemanticNode {
      * Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
      * Ignores Entity with ignored == true or removed == true.
      *
-     * @param type       the type of entities to search for
-     * @param textEntity the entity, which appears in the row to search
+     * @param type            the type of entities to search for
+     * @param redactionEntity the entity, which appears in the row to search
      * @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
      */
-    public List<TextEntity> getEntitiesOfTypeInSameRow(String type, TextEntity textEntity) {
+    public List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
 
-        return textEntity.getIntersectingNodes()
+        return redactionEntity.getIntersectingNodes()
                 .stream()
                 .filter(node -> node instanceof TableCell)
                 .map(node -> (TableCell) node)
@@ -408,9 +305,7 @@ public class Table implements SemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
-                    .map(SemanticNode::getLeafTextBlock)
-                    .collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCell.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
@@ -7,11 +7,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,13 +23,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;
     int col;
@@ -42,10 +36,12 @@ public class TableCell implements GenericSemanticNode {
 
     TextBlock textBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    Set<TextEntity> entities = new HashSet<>();
+    @EqualsAndHashCode.Exclude
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java

@@ -1,24 +1,21 @@
-package com.iqser.red.service.redaction.v1.server.model.document.textblock;
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
-import java.text.BreakIterator;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 
@@ -41,9 +38,8 @@ public class AtomicTextBlock implements TextBlock {
     Page page;
 
     //string coordinates
-    TextRange textRange;
+    Boundary boundary;
     String searchText;
-    List<String> words;
     List<Integer> lineBreaks;
 
     //position coordinates
@@ -65,7 +61,7 @@ public class AtomicTextBlock implements TextBlock {
 
         return AtomicTextBlock.builder()
                 .id(textBlockIdx)
-                .textRange(new TextRange(stringOffset, stringOffset))
+                .boundary(new Boundary(stringOffset, stringOffset))
                 .searchText("")
                 .lineBreaks(Collections.emptyList())
                 .page(page)
@@ -86,7 +82,7 @@ public class AtomicTextBlock implements TextBlock {
                 .id(atomicTextBlockData.getId())
                 .numberOnPage(atomicTextBlockData.getNumberOnPage())
                 .page(page)
-                .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
+                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                 .searchText(atomicTextBlockData.getSearchText())
                 .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
                 .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
@@ -102,34 +98,20 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public TextRange getLineTextRange(int lineNumber) {
+    public Boundary getLineBoundary(int lineNumber) {
 
         if (lineNumber >= numberOfLines() || lineNumber < 0) {
-            return new TextRange(textRange.start(), textRange.start());
+            return new Boundary(boundary.start(), boundary.start());
         }
         if (numberOfLines() == 1) {
-            return textRange;
+            return boundary;
         }
         if (lineNumber == 0) {
-            return new TextRange(textRange.start(), lineBreaks.get(0) + textRange.start());
+            return new Boundary(boundary.start(), lineBreaks.get(0) + boundary.start());
         } else if (lineNumber == numberOfLines() - 1) {
-            return new TextRange(lineBreaks.get(lineBreaks.size() - 1) + textRange.start(), textRange.end());
+            return new Boundary(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end());
         }
-        return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
-    }
-
-    public List<String> getWords() {
-
-        if (words == null) {
-            words = new ArrayList<>();
-            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
-            iterator.setText(searchText);
-            int start = iterator.first();
-            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
-                words.add(searchText.substring(start, end));
-            }
-        }
-        return words;
+        return new Boundary(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start());
     }
 
 
@@ -144,9 +126,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getNextLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
+                .filter(linebreak -> linebreak > fromIndex - boundary.start()) //
                 .findFirst() //
-                .orElse(searchText.length()) + textRange.start();
+                .orElse(searchText.length()) + boundary.start();
     }
 
 
@@ -154,43 +136,43 @@ public class AtomicTextBlock implements TextBlock {
     public int getPreviousLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
+                .filter(linebreak -> linebreak <= fromIndex - boundary.start())//
                 .reduce((a, b) -> b)//
-                .orElse(0) + textRange.start();
+                .orElse(0) + boundary.start();
     }
 
 
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
-        return positions.get(stringIdxToPositionIdx.get(stringIdx - textRange.start()));
+        return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start()));
     }
 
 
     @Override
-    public List<Rectangle2D> getPositions(TextRange stringTextRange) {
+    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
 
-        if (!containsTextRange(stringTextRange)) {
-            throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringTextRange, this.textRange));
+        if (!containsBoundary(stringBoundary)) {
+            throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringBoundary, this.boundary));
         }
-        if (stringTextRange.length() == 0) {
+        if (stringBoundary.length() == 0) {
             return Collections.emptyList();
         }
 
-        int startPositionIdx = stringIdxToPositionIdx.get(stringTextRange.start() - this.textRange.start());
+        int startPositionIdx = stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start());
 
-        if (stringTextRange.end() == this.textRange.end()) {
+        if (stringBoundary.end() == this.boundary.end()) {
             return positions.subList(startPositionIdx, positions.size());
         }
 
-        return positions.subList(startPositionIdx, stringIdxToPositionIdx.get(stringTextRange.end() - this.textRange.start()));
+        return positions.subList(startPositionIdx, stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start()));
 
     }
 
 
-    public Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange) {
+    public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
 
-        List<Rectangle2D> rectanglesPerLine = stringTextRange.split(getAllLineBreaksInBoundary(stringTextRange))
+        List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getAllLineBreaksInBoundary(stringBoundary))
                 .stream()
                 .map(this::getPositions)
                 .map(RectangleTransformations::rectangleBBoxWithGaps)
@@ -203,18 +185,18 @@ public class AtomicTextBlock implements TextBlock {
 
 
     @Override
-    public String subSequenceWithLineBreaks(TextRange textRange) {
+    public String subSequenceWithLineBreaks(Boundary boundary) {
 
-        if (textRange.length() == 0 || !getTextRange().contains(textRange)) {
+        if (boundary.length() == 0 || !getBoundary().contains(boundary)) {
             return "";
         }
 
-        Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + textRange.start()).filter(textRange::contains).collect(Collectors.toSet());
-        if (textRange.end() == getTextRange().end()) {
-            lbInBoundary.add(getTextRange().end());
+        Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + boundary.start()).filter(boundary::contains).collect(Collectors.toSet());
+        if (boundary.end() == getBoundary().end()) {
+            lbInBoundary.add(getBoundary().end());
         }
         StringBuilder sb = new StringBuilder();
-        for (int i = textRange.start(); i < textRange.end(); i++) {
+        for (int i = boundary.start(); i < boundary.end(); i++) {
             char character = this.charAt(i);
             if (lbInBoundary.contains(i + 1)) {
                 // always plus one, due to the linebreaks being an exclusive end index
@@ -233,9 +215,9 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
+    private List<Integer> getAllLineBreaksInBoundary(Boundary boundary) {
 
-        return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
+        return getLineBreaks().stream().map(linebreak -> linebreak + this.boundary.start()).filter(boundary::contains).toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.textblock;
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -7,12 +7,11 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.Data;
@@ -24,7 +23,7 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     List<AtomicTextBlock> atomicTextBlocks;
     String searchText;
-    TextRange textRange;
+    Boundary boundary;
 
 
     public static ConcatenatedTextBlock empty() {
@@ -37,12 +36,12 @@ public class ConcatenatedTextBlock implements TextBlock {
 
         this.atomicTextBlocks = new LinkedList<>();
         if (atomicTextBlocks.isEmpty()) {
-            textRange = new TextRange(-1, -1);
+            boundary = new Boundary(-1, -1);
             return;
         }
         var firstTextBlock = atomicTextBlocks.get(0);
         this.atomicTextBlocks.add(firstTextBlock);
-        textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end());
+        boundary = new Boundary(firstTextBlock.getBoundary().start(), firstTextBlock.getBoundary().end());
 
         atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
     }
@@ -51,13 +50,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     public ConcatenatedTextBlock concat(TextBlock textBlock) {
 
         if (this.atomicTextBlocks.isEmpty()) {
-            textRange.setStart(textBlock.getTextRange().start());
-            textRange.setEnd(textBlock.getTextRange().end());
-        } else if (textRange.end() != textBlock.getTextRange().start()) {
-            throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", textRange, textBlock.getTextRange()));
+            boundary.setStart(textBlock.getBoundary().start());
+            boundary.setEnd(textBlock.getBoundary().end());
+        } else if (boundary.end() != textBlock.getBoundary().start()) {
+            throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", boundary, textBlock.getBoundary()));
         }
         this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
-        textRange.setEnd(textBlock.getTextRange().end());
+        boundary.setEnd(textBlock.getBoundary().end());
         this.searchText = null;
         return this;
     }
@@ -65,22 +64,13 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
 
-        return atomicTextBlocks.stream().filter(textBlock -> textBlock.getTextRange().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
+        return atomicTextBlocks.stream().filter(textBlock -> textBlock.getBoundary().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
     }
 
 
-    private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
+    private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(Boundary boundary) {
 
-        List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
-        for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
-            if (atomicTextBlock.getTextRange().start() > textRange.end()) {
-                break; // early stop, following TextBlocks will never intersect
-            }
-            if (atomicTextBlock.getTextRange().intersects(textRange)) {
-                intersectingAtomicTextBlocks.add(atomicTextBlock);
-            }
-        }
-        return intersectingAtomicTextBlocks;
+        return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList();
     }
 
 
@@ -96,13 +86,6 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
-    @Override
-    public List<String> getWords() {
-
-        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
-    }
-
-
     @Override
     public int numberOfLines() {
 
@@ -131,106 +114,106 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
         return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
     }
 
-
-    public TextRange getLineTextRange(int lineNumber) {
+    public Boundary getLineBoundary(int lineNumber) {
 
         if (atomicTextBlocks.size() == 1) {
-            return atomicTextBlocks.get(0).getLineTextRange(lineNumber);
+            return atomicTextBlocks.get(0).getLineBoundary(lineNumber);
         }
         int lineNumberInCurrentBlock = lineNumber;
         for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
             if (lineNumberInCurrentBlock < atomicTextBlock.numberOfLines()) {
-                return atomicTextBlock.getLineTextRange(lineNumberInCurrentBlock);
+                return atomicTextBlock.getLineBoundary(lineNumberInCurrentBlock);
             }
             lineNumberInCurrentBlock -= atomicTextBlock.numberOfLines();
         }
-        return new TextRange(textRange.start(), textRange.start());
+        return new Boundary(boundary.start(), boundary.start());
     }
 
-
     @Override
-    public List<Rectangle2D> getPositions(TextRange stringTextRange) {
+    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
 
-        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
+
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
 
         if (textBlocks.isEmpty()) {
             return Collections.emptyList();
         }
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).getPositions(stringTextRange);
+            return textBlocks.get(0).getPositions(stringBoundary);
         }
 
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new TextRange(stringTextRange.start(), firstTextBlock.getTextRange().end())));
+        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
             positions.addAll(textBlock.getPositions());
         }
 
         var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        positions.addAll(lastTextBlock.getPositions(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end())));
+        positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
         return positions;
     }
 
 
     @Override
-    public Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange) {
+    public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
 
-        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
 
         if (textBlocks.isEmpty()) {
             return new HashMap<>();
         }
 
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).getPositionsPerPage(stringTextRange);
+            return textBlocks.get(0).getPositionsPerPage(stringBoundary);
         }
 
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getPositionsPerPage(new TextRange(stringTextRange.start(), firstTextBlock.getTextRange().end()));
+        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
-            rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getPositionsPerPage(textBlock.getTextRange()));
+            rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getPositionsPerPage(textBlock.getBoundary()));
         }
 
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
         rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage,
-                lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end())));
+                lastTextBlock.getPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
         return rectanglesPerLinePerPage;
     }
 
 
     @Override
-    public String subSequenceWithLineBreaks(TextRange textRange) {
+    public String subSequenceWithLineBreaks(Boundary boundary) {
 
-        if (textRange.length() == 0 || !getTextRange().contains(textRange)) {
+        if (boundary.length() == 0 || !getBoundary().contains(boundary)) {
             return "";
         }
 
-        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(textRange);
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(boundary);
 
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).subSequenceWithLineBreaks(textRange);
+            return textBlocks.get(0).subSequenceWithLineBreaks(boundary);
         }
 
         StringBuilder sb = new StringBuilder();
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        sb.append(firstTextBlock.subSequenceWithLineBreaks(new TextRange(textRange.start(), firstTextBlock.getTextRange().end())));
+        sb.append(firstTextBlock.subSequenceWithLineBreaks(new Boundary(boundary.start(), firstTextBlock.getBoundary().end())));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
             sb.append(textBlock.searchTextWithLineBreaks());
         }
 
         var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        sb.append(lastTextBlock.subSequenceWithLineBreaks(new TextRange(lastTextBlock.getTextRange().start(), textRange.end())));
+        sb.append(lastTextBlock.subSequenceWithLineBreaks(new Boundary(lastTextBlock.getBoundary().start(), boundary.end())));
 
         return sb.toString();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.textblock;
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -10,20 +10,19 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
 
 public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
-    List<String> getWords();
 
     List<AtomicTextBlock> getAtomicTextBlocks();
 
 
-    TextRange getTextRange();
+    Boundary getBoundary();
 
 
     int getNextLinebreak(int fromIndex);
@@ -32,7 +31,7 @@ public interface TextBlock extends CharSequence {
     int getPreviousLinebreak(int fromIndex);
 
 
-    TextRange getLineTextRange(int lineNumber);
+    Boundary getLineBoundary(int lineNumber);
 
 
 
@@ -42,13 +41,13 @@ public interface TextBlock extends CharSequence {
     Rectangle2D getPosition(int stringIdx);
 
 
-    List<Rectangle2D> getPositions(TextRange stringTextRange);
+    List<Rectangle2D> getPositions(Boundary stringBoundary);
 
 
-    Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange);
+    Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary);
 
 
-    String subSequenceWithLineBreaks(TextRange textRange);
+    String subSequenceWithLineBreaks(Boundary boundary);
 
 
     int numberOfLines();
@@ -56,13 +55,13 @@ public interface TextBlock extends CharSequence {
 
     default CharSequence getLine(int lineNumber) {
 
-        return subSequence(getLineTextRange(lineNumber));
+        return subSequence(getLineBoundary(lineNumber));
     }
 
 
     default List<Rectangle2D> getLinePositions(int lineNumber) {
 
-        return getPositions(getLineTextRange(lineNumber));
+        return getPositions(getLineBoundary(lineNumber));
     }
 
 
@@ -73,13 +72,13 @@ public interface TextBlock extends CharSequence {
 
     default String searchTextWithLineBreaks() {
 
-        return subSequenceWithLineBreaks(getTextRange());
+        return subSequenceWithLineBreaks(getBoundary());
     }
 
 
     default int indexOf(String searchTerm) {
 
-        return indexOf(searchTerm, getTextRange().start());
+        return indexOf(searchTerm, getBoundary().start());
     }
 
 
@@ -89,10 +88,10 @@ public interface TextBlock extends CharSequence {
     }
 
 
-    default Set<Page> getPages(TextRange textRange) {
+    default Set<Page> getPages(Boundary boundary) {
 
         return getAtomicTextBlocks().stream()
-                .filter(atomicTextBlock -> atomicTextBlock.getTextRange().intersects(textRange))
+                .filter(atomicTextBlock -> atomicTextBlock.getBoundary().intersects(boundary))
                 .map(AtomicTextBlock::getPage)
                 .collect(Collectors.toUnmodifiableSet());
     }
@@ -100,38 +99,38 @@ public interface TextBlock extends CharSequence {
 
     default int indexOf(String searchTerm, int startOffset) {
 
-        int start = getSearchText().indexOf(searchTerm, startOffset - getTextRange().start());
+        int start = getSearchText().indexOf(searchTerm, startOffset - getBoundary().start());
         if (start == -1) {
             return -1;
         }
-        return start + getTextRange().start();
+        return start + getBoundary().start();
     }
 
 
     default CharSequence getFirstLine() {
 
-        return subSequence(getTextRange().start(), getNextLinebreak(getTextRange().start()));
+        return subSequence(getBoundary().start(), getNextLinebreak(getBoundary().start()));
     }
 
 
-    default boolean containsTextRange(TextRange textRange) {
+    default boolean containsBoundary(Boundary boundary) {
 
-        if (textRange.end() < textRange.start()) {
-            throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", textRange));
+        if (boundary.end() < boundary.start()) {
+            throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", boundary));
         }
-        return getTextRange().contains(textRange);
+        return getBoundary().contains(boundary);
     }
 
 
     default boolean containsIndex(int stringIndex) {
 
-        return getTextRange().contains(stringIndex);
+        return getBoundary().contains(stringIndex);
     }
 
 
-    default CharSequence subSequence(TextRange textRange) {
+    default CharSequence subSequence(Boundary boundary) {
 
-        return subSequence(textRange.start(), textRange.end());
+        return subSequence(boundary.start(), boundary.end());
     }
 
 
@@ -148,21 +147,21 @@ public interface TextBlock extends CharSequence {
     @Override
     default CharSequence subSequence(int start, int end) {
 
-        return getSearchText().substring(start - getTextRange().start(), end - getTextRange().start());
+        return getSearchText().substring(start - getBoundary().start(), end - getBoundary().start());
     }
 
 
     @Override
     default int length() {
 
-        return getTextRange().length();
+        return getBoundary().length();
     }
 
 
     @Override
     default char charAt(int index) {
 
-        return getSearchText().charAt(index - getTextRange().start());
+        return getSearchText().charAt(index - getBoundary().start());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.model.document.textblock;
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 
 import java.util.Set;
 import java.util.function.BiConsumer;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java

@@ -0,0 +1,670 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.kie.api.runtime.KieSession;
+
+import com.google.common.base.Functions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.ConsecutiveBoundaryCollector;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class EntityCreationService {
+
+    private final EntityEnrichmentService entityEnrichmentService;
+    private final KieSession kieSession;
+
+
+    public EntityCreationService(EntityEnrichmentService entityEnrichmentService) {
+
+        this.entityEnrichmentService = entityEnrichmentService;
+        this.kieSession = null;
+    }
+
+
+    public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
+
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
+
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
+
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
+
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
+
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
+        });
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+
+
+
+    public Stream<RedactionEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
+
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
+        });
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
+        });
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+
+
+
+    public Stream<RedactionEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
+
+        TextBlock textBlock = node.getTextBlock();
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStart, textBlock);
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStop, textBlock);
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
+
+        TextBlock textBlock = node.getTextBlock();
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStart, 0, textBlock);
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStop, 0, textBlock);
+
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
+    }
+
+
+    public Stream<RedactionEntity> betweenBoundaries(List<Boundary> startBoundaries, List<Boundary> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
+
+        if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
+            return Stream.empty();
+        }
+        List<Boundary> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
+        return entityBoundaries.stream()
+                .map(boundary -> boundary.trim(node.getTextBlock()))
+                .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    private static List<Boundary> findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(List<Boundary> startBoundaries, List<Boundary> stopBoundaries) {
+
+        List<Boundary> entityBoundaries = new LinkedList<>();
+        for (Boundary startBoundary : startBoundaries) {
+            Optional<Boundary> optionalStopBoundaryWithMinimalDistance = stopBoundaries.stream()
+                    .filter(stopBoundary -> stopBoundary.start() > startBoundary.end())
+                    .min(Comparator.comparingInt(Boundary::start));
+            if (optionalStopBoundaryWithMinimalDistance.isEmpty()) {
+                break;
+            }
+            entityBoundaries.add(new Boundary(startBoundary.end(), optionalStopBoundaryWithMinimalDistance.get().start()));
+        }
+        return removeOuterOverlappingBoundaries(entityBoundaries);
+    }
+
+
+    private static List<Boundary> removeOuterOverlappingBoundaries(List<Boundary> entityBoundaries) {
+        /*
+        In some cases we get boundaries, where one contains the other. This happens for Example when we have two start boundaries and one stop boundary after the two start boundaries.
+        Then we get two boundaries where one is entirely contained in the other. So we want to remove the outer boundary.
+        For Example consider the text: "a this is some text. a here is more text b". If "a" is the start string and "b" is the stop string, there are two possibilities.
+        "this is some text. a here is more text" and "here is more text". We only want to keep the latter.
+         */
+        return entityBoundaries.stream()
+                .filter(boundary -> entityBoundaries.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary)))
+                .toList();
+
+    }
+
+
+    public Stream<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
+                .stream()
+                .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
+                .map(bounds -> byBoundary(bounds, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
+
+        TextBlock textBlock = node.getTextBlock();
+        SearchImplementation searchImplementation = new SearchImplementation(strings, false);
+        return searchImplementation.getBoundaries(textBlock, node.getBoundary())
+                .stream()
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
+
+        TextBlock textBlock = node.getTextBlock();
+        return RedactionSearchUtility.findBoundariesByString(string, textBlock)
+                .stream()
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
+
+        return tableNode.streamTableCells()
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByString(string, tableCell.getTextBlock()),
+                        tableCell,
+                        type,
+                        entityType,
+                        tableNode));
+    }
+
+
+    public Stream<RedactionEntity> lineAfterStringAcrossColumnsIgnoreCase(String string, String type, EntityType entityType, Table tableNode) {
+
+        return tableNode.streamTableCells()
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, tableCell.getTextBlock()),
+                        tableCell,
+                        type,
+                        entityType,
+                        tableNode));
+    }
+
+
+    /**
+     * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
+     *
+     * @param boundaries a list of boundaries
+     * @param tableCell  the table cell
+     * @param type       the type
+     * @param entityType the entity type
+     * @param tableNode  the table node
+     * @return a stream of RedactionEntities
+     */
+    private Stream<RedactionEntity> lineAfterBoundariesAcrossColumns(List<Boundary> boundaries, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
+
+        return boundaries.stream()
+                .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
+                .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
+                .map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
+                        .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
+                        .map(nextTableCell -> RedactionSearchUtility.findBoundaryOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
+                        .map(b -> b.trim(tableNode.getTextBlock()))
+                        .filter(boundary -> isValidEntityBoundary(tableNode.getTextBlock(), boundary))
+                        .map(boundary -> byBoundary(boundary, type, entityType, tableNode))
+                        .filter(Optional::isPresent)
+                        .map(Optional::get))
+                .flatMap(Functions.identity());
+    }
+
+
+    public Optional<RedactionEntity> semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) {
+
+        var textBlock = semanticNode.getTextBlock();
+        int startIndex = Math.min(textBlock.indexOf(string), 0);
+        var boundary = new Boundary(startIndex, semanticNode.getBoundary().end());
+        if (boundary.length() > 0) {
+            boundary = new Boundary(boundary.start(), boundary.end() - 1);
+        }
+        if (!isValidEntityBoundary(textBlock, boundary)) {
+            return Optional.empty();
+        }
+        return byBoundary(boundary, type, entityType, semanticNode);
+    }
+
+
+    public Stream<RedactionEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) {
+
+        return byRegexWithLineBreaks(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) {
+
+        return byRegexWithLineBreaksIgnoreCase(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
+
+        return byRegex(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) {
+
+        return byRegexIgnoreCase(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) {
+
+        return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get);
+    }
+
+
+    public Stream<RedactionEntity> bySemanticNodeParagraphsOnlyMergeConsecutive(SemanticNode node, String type, EntityType entityType) {
+
+        return node.streamAllSubNodesOfType(NodeType.PARAGRAPH)
+                .map(SemanticNode::getBoundary)
+                .collect(new ConsecutiveBoundaryCollector())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+    public Optional<RedactionEntity> semanticNodeAfterString(String string, String type, EntityType entityType, SemanticNode node) {
+
+        if (!node.containsString(string)) {
+            return Optional.empty();
+        }
+        Boundary boundary = new Boundary(node.getTextBlock().indexOf(string) + string.length(), node.getBoundary().end());
+        return byBoundary(boundary, type, entityType, node);
+    }
+
+    public Optional<RedactionEntity> bySemanticNode(SemanticNode node, String type, EntityType entityType) {
+
+        Boundary boundary = node.getTextBlock().getBoundary();
+
+        if (boundary.length() > 0) {
+            boundary = new Boundary(boundary.start(), boundary.end() - 1);
+        }
+        if (!isValidEntityBoundary(node.getTextBlock(), boundary)) {
+            return Optional.empty();
+        }
+        return byBoundary(boundary, type, entityType, node);
+    }
+
+
+    public Optional<RedactionEntity> byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
+        return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+    }
+
+
+    public Optional<RedactionEntity> bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
+        expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
+        return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+    }
+
+
+    private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+
+        if (textBlock.getNextLinebreak(end) < expandedEnd) {
+            return end;
+        }
+        return expandedEnd;
+    }
+
+
+    /**
+     * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
+     * If the document already contains an equal redaction entity, then the original Entity is returned.
+     * Also inserts the Entity into the kieSession.
+     *
+     * @param boundary   The boundary of the redaction entity.
+     * @param type       The type of the redaction entity.
+     * @param entityType The entity type of the redaction entity.
+     * @param node       The semantic node to associate with the redaction entity.
+     * @return An Optional containing the redaction entity, or the previous entity if the entity already exists.
+     */
+    public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
+
+        if (!node.getBoundary().contains(boundary)) {
+            throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", boundary, node.getBoundary(), node));
+        }
+        Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
+        RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
+        if (node.getEntities().contains(entity)) {
+            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny();
+        }
+        addEntityToGraph(entity, node);
+        entity.addEngine(Engine.RULE);
+        insertToKieSession(entity);
+        return Optional.of(entity);
+    }
+
+
+    public RedactionEntity forceByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
+
+        Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
+        RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
+        addEntityToGraph(entity, node);
+        return entity;
+    }
+
+
+    public RedactionEntity byEntities(List<RedactionEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
+
+        if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
+            throw new IllegalArgumentException("Provided entities can not be merged, since they do not intersect or are not the same type!" + entitiesToMerge);
+        }
+        if (entitiesToMerge.isEmpty()) {
+            throw new IllegalArgumentException("No Entities to merge!");
+        }
+        if (entitiesToMerge.size() == 1) {
+            return entitiesToMerge.get(0);
+        }
+
+        RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
+        mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
+        entitiesToMerge.stream().map(RedactionEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
+
+        mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry));
+        mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry));
+        mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored));
+        mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
+
+        addEntityToGraph(mergedEntity, node);
+        insertToKieSession(mergedEntity);
+        return mergedEntity;
+    }
+
+
+    private void insertToKieSession(RedactionEntity mergedEntity) {
+
+        if (kieSession != null) {
+            kieSession.insert(mergedEntity);
+        }
+    }
+
+
+    public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
+
+        var entity = forceByBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
+        entity.addEngine(Engine.NER);
+        insertToKieSession(entity);
+        return entity;
+    }
+
+
+    public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
+
+        var entity = forceByBoundary(nerEntity.boundary(), type, entityType, semanticNode);
+        entity.addEngine(Engine.NER);
+        insertToKieSession(entity);
+        return entity;
+    }
+
+
+    public Stream<RedactionEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
+
+        return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
+                .map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
+                .peek(entity -> entity.addEngine(Engine.NER))
+                .peek(this::insertToKieSession);
+    }
+
+
+    public RedactionEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
+
+        RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCell.getBoundary().start(), tableCell.getBoundary().start()), type, entityType);
+
+        String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
+        highlightEntity.setRedactionPositionsPerPage(tableCell.getBBox()
+                .entrySet()
+                .stream()
+                .map(entry -> new RedactionPosition(positionId, entry.getKey(), List.of(entry.getValue())))
+                .toList());
+        addEntityToGraph(highlightEntity, tableCell);
+        return highlightEntity;
+    }
+
+
+    public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
+
+        return boundary.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, boundary);
+    }
+
+
+    public void addEntityToGraph(RedactionEntity entity, SemanticNode node) {
+
+        DocumentTree documentTree = node.getDocumentTree();
+        try {
+            addEntityToGraph(entity, documentTree);
+            entity.addIntersectingNode(documentTree.getRoot().getNode());
+            documentTree.getRoot().getNode().getEntities().add(entity);
+        } catch (NoSuchElementException e) {
+            entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
+            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
+            entity.addIntersectingNode(documentTree.getRoot().getNode());
+            addToPages(entity);
+            addEntityToNodeEntitySets(entity);
+        }
+    }
+
+
+    private void addEntityToGraph(RedactionEntity entity, DocumentTree documentTree) {
+
+        SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
+                .filter(node -> node.getTextBlock().containsBoundary(entity.getBoundary()))
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
+
+        containingNode.addThisToEntityIfIntersects(entity);
+
+        TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
+        entityEnrichmentService.enrichEntity(entity, textBlock);
+
+        addToPages(entity);
+        addEntityToNodeEntitySets(entity);
+    }
+
+
+    private static void addToPages(RedactionEntity entity) {
+
+        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary());
+        entity.getPages().addAll(pages);
+        pages.forEach(page -> page.getEntities().add(entity));
+    }
+
+
+    private static void addEntityToNodeEntitySets(RedactionEntity entity) {
+
+        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
+    }
+
+
+    private static boolean allEntitiesIntersectAndHaveSameTypes(List<RedactionEntity> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        RedactionEntity previousEntity = entitiesToMerge.get(0);
+        for (RedactionEntity redactionEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = redactionEntity.getType().equals(previousEntity.getType());
+            boolean entityTypeMatches = redactionEntity.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = redactionEntity.intersects(previousEntity);
+            if (!typeMatches || !entityTypeMatches || !intersects) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
+
+        return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())).trim(textBlock);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
+package com.iqser.red.service.redaction.v1.server.document.services;
 
 import java.util.Arrays;
 import java.util.List;
@@ -6,9 +6,9 @@ import java.util.Objects;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import lombok.RequiredArgsConstructor;
 
@@ -19,17 +19,17 @@ public class EntityEnrichmentService {
     private final RedactionServiceSettings redactionServiceSettings;
 
 
-    public void enrichEntity(TextEntity entity, TextBlock textBlock) {
+    public void enrichEntity(RedactionEntity entity, TextBlock textBlock) {
 
-        entity.setValue(textBlock.subSequence(entity.getTextRange()).toString());
-        entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock));
-        entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock));
+        entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
+        entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
+        entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
     }
 
 
     private String findTextAfter(int index, TextBlock textBlock) {
 
-        int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end());
+        int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().end());
         String textAfter = textBlock.subSequence(index, endOffset).toString();
         if (!textAfter.isBlank()) {
             List<String> wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter);
@@ -44,7 +44,7 @@ public class EntityEnrichmentService {
 
     private String findTextBefore(int index, TextBlock textBlock) {
 
-        int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start());
+        int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start());
         String textBefore = textBlock.subSequence(offsetBefore, index).toString();
         if (!textBefore.isBlank()) {
             List<String> wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualRedactionApplicationService.java

@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.awt.geom.Rectangle2D;
+import java.util.NoSuchElementException;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
+
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class ManualRedactionApplicationService {
+
+    private final EntityCreationService entityCreationService;
+
+
+    public void resizeEntityAndReinsert(RedactionEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
+
+        RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
+
+        redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
+
+        int newStartOffset;
+        if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
+            newStartOffset = entityToBeResized.getBoundary().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
+        } else {
+            newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
+        }
+
+        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
+        entityToBeResized.removeFromGraph();
+        entityToBeResized.setResized(true);
+        entityToBeResized.setRemoved(false);
+        entityToBeResized.setIgnored(false);
+        entityToBeResized.getBoundary().setStart(newStartOffset);
+        entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
+        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
+    }
+
+
+    public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
+
+        if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {
+            return;
+        }
+        var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
+        image.setPosition(bBox);
+    }
+
+
+    private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
+
+        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RectangleTransformations.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.utils;
+package com.iqser.red.service.redaction.v1.server.document.utils;
 
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
@@ -12,23 +12,44 @@ import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collector;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;
 
 public class RectangleTransformations {
 
+
     public static Rectangle2D atomicTextBlockBBox(List<AtomicTextBlock> atomicTextBlocks) {
 
         return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
     }
 
 
-    public static Rectangle2D rectangleBBox(List<Position> positions) {
+    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {
 
-        return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector());
+        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
+    }
+
+
+    public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
+
+        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
+                redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(),
+                redactionLogRectangle.getWidth(),
+                -redactionLogRectangle.getHeight());
+    }
+
+
+    public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 
@@ -70,12 +91,6 @@ public class RectangleTransformations {
     }
 
 
-    public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {
-
-        return new Rectangle2DBBoxCollector();
-    }
-
-
     private static class Rectangle2DBBoxCollector implements Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RedactionSearchUtility.java

@@ -0,0 +1,182 @@
+package com.iqser.red.service.redaction.v1.server.document.utils;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.IntStream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class RedactionSearchUtility {
+
+    public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return pattern.matcher(charSequence).find();
+    }
+
+
+    public static boolean anyMatchIgnoreCase(CharSequence charSequence, String regexPattern) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
+        return pattern.matcher(charSequence).find();
+    }
+
+
+    public static boolean exactMatch(CharSequence charSequence, String regexPattern) {
+
+        return charSequence.toString().matches(regexPattern);
+    }
+
+
+    public static boolean anyMatchIgnoreCase(TextBlock textBlock, String regexPattern) {
+
+        return anyMatchIgnoreCase(textBlock.getSearchText(), regexPattern);
+    }
+
+
+    public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
+
+        return anyMatch(textBlock.getSearchText(), regexPattern);
+    }
+
+
+    public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Matcher matcher = pattern.matcher(searchText);
+        if (matcher.find()) {
+            return new Boundary(matcher.start(), matcher.end());
+        }
+        throw new IllegalArgumentException(format("Charsequence %s does not contain any matches for pattern %s", searchText, regexPattern));
+    }
+
+
+    public static int getExpandedEndByRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedEnd;
+        if (anyMatch(entity.getTextAfter(), regexPattern)) {
+            Boundary postfixBoundary = findFirstBoundary(regexPattern, entity.getTextAfter());
+            expandedEnd = postfixBoundary.end() + entity.getBoundary().end();
+        } else {
+            expandedEnd = entity.getBoundary().end();
+        }
+        return expandedEnd;
+    }
+
+
+    public static int getExpandedStartByRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedStart;
+        if (anyMatch(entity.getTextBefore(), regexPattern)) {
+            Boundary prefixBoundary = findFirstBoundary(regexPattern, entity.getTextBefore());
+            expandedStart = prefixBoundary.start() + entity.getBoundary().start() - entity.getTextBefore().length();
+        } else {
+            expandedStart = entity.getBoundary().start();
+        }
+        return expandedStart;
+    }
+
+    public static Boundary findBoundaryOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {
+
+        List<Boundary> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed().map(textBlock::getLineBoundary).filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)).toList();
+        if (lineBoundaries.isEmpty()) {
+            return new Boundary(textBlock.getBoundary().start(), textBlock.getBoundary().start());
+        }
+        return Boundary.merge(lineBoundaries);
+    }
+
+
+    private static boolean isWithinYRange(double maxY, double minY, TextBlock textBlock, Boundary lineBoundary) {
+
+        Rectangle2D lineBBox = RectangleTransformations.rectangle2DBBox(textBlock.getPositions(lineBoundary));
+        return lineBBox.getMinY() < maxY && minY < lineBBox.getMaxY();
+    }
+
+
+    public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return getBoundariesByPattern(textBlock, 0, pattern);
+
+    }
+
+
+    public static List<Boundary> findBoundariesByRegex(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return getBoundariesByPattern(textBlock, group, pattern);
+    }
+
+
+    public static List<Boundary> findBoundariesByRegexWithLineBreaks(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, false);
+        return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern);
+    }
+
+
+    public static List<Boundary> findBoundariesByRegexWithLineBreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, true);
+        return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern);
+    }
+
+
+    public static List<Boundary> findBoundariesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
+        return getBoundariesByPattern(textBlock, group, pattern);
+    }
+
+
+    private static List<Boundary> getBoundariesByPattern(TextBlock textBlock, int group, Pattern pattern) {
+
+        Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
+        List<Boundary> boundaries = new LinkedList<>();
+        while (matcher.find()) {
+            boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start()));
+        }
+        return boundaries;
+    }
+
+
+    private static List<Boundary> getBoundariesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) {
+
+        String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks();
+        Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
+        List<Boundary> boundaries = new LinkedList<>();
+        while (matcher.find()) {
+            boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start()));
+        }
+        return boundaries;
+    }
+
+
+    public static List<Boundary> findBoundariesByString(String searchString, TextBlock textBlock) {
+
+        List<Boundary> boundaries = new LinkedList<>();
+        for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) {
+            boundaries.add(new Boundary(index, index + searchString.length()));
+        }
+        return boundaries;
+    }
+
+
+    public static List<Boundary> findBoundariesByStringIgnoreCase(String searchString, TextBlock textBlock) {
+
+        Pattern pattern = Pattern.compile(Pattern.quote(searchString), Pattern.CASE_INSENSITIVE);
+        return getBoundariesByPattern(textBlock, 0, pattern);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/NotFoundException.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.utils.exception;
+package com.iqser.red.service.redaction.v1.server.exception;
 
 public class NotFoundException extends RuntimeException {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RedactionException.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.utils.exception;
+package com.iqser.red.service.redaction.v1.server.exception;
 
 public class RedactionException extends RuntimeException {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RulesValidationException.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.utils.exception;
+package com.iqser.red.service.redaction.v1.server.exception;
 
 public class RulesValidationException extends RuntimeException {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -1,428 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LegacyRedactionLogMergeService {
-
-    private final DictionaryService dictionaryService;
-
-
-    public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
-
-        Set<String> skippedImportedRedactions = new HashSet<>();
-        log.info("Adding manual add Entries and removing skipped or imported entries");
-        if (manualRedactions != null) {
-
-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                if (entry.isImported()) {
-                    processRedactionLogEntry(manualRedactionWrappers.stream()
-                                                     .filter(ManualRedactionWrapper::isApproved)
-                                                     .filter(mr -> entry.getId().equals(mr.getId()))
-                                                     .collect(Collectors.toList()), entry, dossierTemplateId);
-                    if (!entry.isRedacted()) {
-                        skippedImportedRedactions.add(entry.getId());
-                    }
-                }
-
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-            if (entry.isFalsePositive()) {
-                return true;
-            }
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
-
-        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
-    }
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getRecategorizations()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getIdsToRemove()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getForceRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getLegalBasisChanges()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getResizeRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            Object item = mrw.getItem();
-            if (item instanceof ManualRecategorization imageRecategorization) {
-                processManualImageRecategorization(redactionLogEntry, dossierTemplateId, imageRecategorization);
-            }
-
-            if (item instanceof IdRemoval manualRemoval) {
-                processIdRemoval(redactionLogEntry, manualRemoval);
-            }
-
-            if (item instanceof ManualForceRedaction manualForceRedact) {
-                processManualForceRedaction(redactionLogEntry, dossierTemplateId, manualForceRedact);
-            }
-
-            if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
-            }
-
-            if (item instanceof ManualResizeRedaction manualResizeRedact) {
-                processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
-            }
-
-        });
-
-    }
-
-
-    private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualRecategorization imageRecategorization) {
-
-        String manualOverrideReason = null;
-        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-            redactionLogEntry.setType(imageRecategorization.getType());
-            redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-            if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
-                redactionLogEntry.setRedacted(false);
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setHint(false);
-            }
-
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-        } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
-
-        boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
-        if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
-            log.debug("Skipping merge for dictionary-modifying entry");
-        } else {
-            String manualOverrideReason = null;
-            if (isApprovedRedaction) {
-                redactionLogEntry.setRedacted(false);
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                redactionLogEntry.setHint(false);
-            } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-            }
-
-            if (manualOverrideReason != null) {
-                redactionLogEntry.setReason(manualOverrideReason);
-            }
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualRemoval)
-                             .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-    }
-
-
-    private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
-
-        return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
-    }
-
-
-    private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualForceRedaction manualForceRedact) {
-
-        String manualOverrideReason = null;
-        var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
-        if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            // Forcing a skipped hint should result in a hint
-            if (dictionaryIsHint) {
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setRedacted(true);
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
-
-        String manualOverrideReason = null;
-        if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-            redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-            redactionLogEntry.setRedacted(true);
-            if (manualLegalBasisChange.getSection() != null) {
-                redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-            }
-            if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-            }
-        } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-        manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-        if (manualLegalBasisChange.getSection() != null) {
-            manualChange.withChange("section", manualLegalBasisChange.getSection());
-        }
-        if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-            manualChange.withChange("value", manualLegalBasisChange.getValue());
-        }
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
-
-        String manualOverrideReason = null;
-        if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-            if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                redactionLogEntry.setValue(manualResizeRedact.getValue());
-            }
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-        } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-        }
-
-        redactionLogEntry.setReason(manualOverrideReason);
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        if (!manualRedactionEntry.isApproved()) {
-            return false;
-        }
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
-                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
-                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-        private boolean approved;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java

@@ -1,70 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.lang.reflect.Field;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-import lombok.SneakyThrows;
-
-@Service
-@SuppressWarnings("PMD")
-public class LegacyVersion0MigrationService {
-
-    public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
-
-        List<RedactionLogEntry> mergedEntries = new LinkedList<>();
-        Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId));
-        for (List<RedactionLogEntry> entries : entriesById.values()) {
-
-            if (entries.isEmpty()) {
-                continue;
-            }
-
-            if (entries.size() == 1) {
-                mergedEntries.add(entries.get(0));
-                continue;
-            }
-
-            List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList();
-
-            RedactionLogEntry initialEntry = sortedEntries.get(0);
-            for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {
-                copyNonNullFields(entry, initialEntry);
-            }
-            mergedEntries.add(initialEntry);
-        }
-
-        redactionLog.setRedactionLogEntry(mergedEntries);
-        return redactionLog;
-    }
-
-
-    @SneakyThrows
-    public static void copyNonNullFields(RedactionLogEntry source, RedactionLogEntry destination) {
-
-        if (source == null || destination == null) {
-            throw new IllegalArgumentException("Source and destination objects must not be null");
-        }
-
-        Class<?> sourceClass = source.getClass();
-
-        Field[] sourceFields = sourceClass.getDeclaredFields();
-
-        for (Field field : sourceFields) {
-            field.setAccessible(true);
-            Object value = field.get(source);
-            if (value != null) {
-                field.set(destination, value);
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -1,93 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-public class MigrationMapper {
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                                                                                                              toEntityLogType(change.getType()),
-                                                                                                              change.getDateTime());
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                                                                                                                    manualChange.getProcessedDate(),
-                                                                                                                    manualChange.getRequestedDate(),
-                                                                                                                    manualChange.getUserId(),
-                                                                                                                    manualChange.getPropertyChanges());
-    }
-
-
-    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
-    }
-
-
-    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        if (entry.getEngines() == null) {
-            return Collections.emptySet();
-        }
-        return entry.getEngines()
-                .stream()
-                .map(MigrationMapper::toEntityLogEngine)
-                .collect(Collectors.toSet());
-    }
-
-
-    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
-
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream()
-                .map(MigrationMapper::toEntityLogManualChanges)
-                .toList();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -1,93 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
-
-import org.springframework.amqp.core.Message;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.service.redaction.v1.model.MigrationResponse;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class MigrationMessageReceiver {
-
-    ObjectMapper objectMapper;
-    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
-    RedactionStorageService redactionStorageService;
-    LegacyRedactionLogMergeService legacyRedactionLogMergeService;
-    LegacyVersion0MigrationService legacyVersion0MigrationService;
-    RabbitTemplate rabbitTemplate;
-    DictionaryService dictionaryService;
-
-
-    @SneakyThrows
-    @RabbitHandler
-    @RabbitListener(queues = MIGRATION_QUEUE)
-    public void receiveMigrationRequest(Message message) {
-
-        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
-        log.info("--------------------------------------------------------------------");
-        log.info("Starting redactionLog to entityLog migration for dossierId {} and fileId {}", migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        dictionaryService.updateDictionary(migrationRequest.getDossierTemplateId(), migrationRequest.getDossierId());
-
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        if (redactionLog.getAnalysisVersion() == 0) {
-            redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else if (migrationRequest.getManualRedactions() != null) {
-            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
-                                                                                                      migrationRequest.getManualRedactions(),
-                                                                                                      migrationRequest.getDossierTemplateId());
-        }
-
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
-                                                                                              document,
-                                                                                              migrationRequest.getDossierTemplateId(),
-                                                                                              migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId());
-
-        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
-
-        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
-                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                 migratedEntityLog.getMigratedIds().getMappings().size(),
-                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
-                 migrationRequest.getDossierId(),
-                 migrationRequest.getFileId());
-        log.info("");
-    }
-
-
-    @SneakyThrows
-    public void sendFinished(MigrationResponse migrationResponse) {
-
-        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -1,290 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-//TODO: remove this, once the migration is done
-public class RedactionLogToEntityLogMigrationService {
-
-    private static final double MATCH_THRESHOLD = 10;
-    EntityFindingUtility entityFindingUtility;
-    EntityEnrichmentService entityEnrichmentService;
-    DictionaryService dictionaryService;
-    ManualChangesApplicationService manualChangesApplicationService;
-
-
-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
-
-        log.info("Migrating entities for file {}", fileId);
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
-
-        MigratedIds migratedIds = entitiesToMigrate.stream()
-                .collect(new MigratedIdsCollector());
-
-        applyManualChanges(entitiesToMigrate, manualRedactions);
-        log.info("applying manual changes to migrated entities for file {}", fileId);
-
-        EntityLog entityLog = new EntityLog();
-        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
-        entityLog.setRulesVersion(redactionLog.getRulesVersion());
-        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
-        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
-        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
-        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
-        entityLog.setLegalBasis(redactionLog.getLegalBasis()
-                                        .stream()
-                                        .map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
-                                        .toList());
-
-        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
-
-        log.info("Writing migrated entities to entityLog for file {}", fileId);
-        entityLog.setEntityLogEntry(entitiesToMigrate.stream()
-                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
-                                            .toList());
-
-        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
-            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
-                                           redactionLog.getRedactionLogEntry().size(),
-                                           entityLog.getEntityLogEntry().size());
-            log.error(message);
-            throw new AssertionError(message);
-        }
-
-        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(MigrationEntity::hasManualChangesOrComments)
-                .filter(m -> !m.getOldId().equals(m.getNewId()))
-                .collect(new MigratedIdsCollector());
-
-        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
-                .filter(MigrationEntity::needsManualEntry)
-                .map(MigrationEntity::buildManualRedactionEntry)
-                .toList();
-
-        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
-
-        return new MigratedEntityLog(idsToMigrateInDb, entityLog);
-    }
-
-
-    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return;
-        }
-
-        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
-                                                                                   manualRedactions.getEntriesToAdd(),
-                                                                                   manualRedactions.getForceRedactions(),
-                                                                                   manualRedactions.getResizeRedactions(),
-                                                                                   manualRedactions.getLegalBasisChanges(),
-                                                                                   manualRedactions.getRecategorizations())
-                .flatMap(Collection::stream)
-                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-
-        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
-                                                                                                                                  Collections.emptyList()),
-                                                                                        manualChangesApplicationService));
-
-    }
-
-
-    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
-
-        return redactionLog.getRedactionLogEntry().size();
-    }
-
-
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
-        return Stream.of(textMigrationEntities.stream(), images.stream())
-                .flatMap(Function.identity())
-                .toList();
-    }
-
-
-    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
-
-        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
-    }
-
-
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
-
-        List<Image> images = document.streamAllImages()
-                .collect(Collectors.toList());
-
-        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(RedactionLogEntry::isImage)
-                .toList();
-
-        List<MigrationEntity> migrationEntities = new LinkedList<>();
-        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
-            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
-                    .stream()
-                    .map(RectangleWithPage::fromRedactionLogRectangle)
-                    .toList();
-            assert imagePositions.size() == 1;
-            Optional<Image> optionalClosestImage = images.stream()
-                    .filter(image -> image.onPage(redactionLogImage.getPositions()
-                                                          .get(0).getPage()))
-                    .min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
-                    .filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
-
-            Image closestImage;
-            if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
-                closestImage = buildImageDirectly(document, redactionLogImage);
-            } else {
-                closestImage = optionalClosestImage.get();
-                images.remove(closestImage);
-            }
-
-            String ruleIdentifier;
-            String reason = Optional.ofNullable(redactionLogImage.getReason())
-                    .orElse("");
-            if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
-                ruleIdentifier = "OLDIMG.0.0";
-            } else {
-                ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
-            }
-
-            if (redactionLogImage.lastChangeIsRemoved()) {
-                closestImage.remove(ruleIdentifier, reason);
-            } else if (redactionLogImage.isRedacted()) {
-                closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
-            } else {
-                closestImage.skip(ruleIdentifier, reason);
-            }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
-        }
-        return migrationEntities;
-    }
-
-
-    private static Image buildImageDirectly(Document document, RedactionLogEntry redactionLogImage) {
-
-        Image image = Image.builder()
-                .documentTree(document.getDocumentTree())
-                .imageType(ImageType.fromString(redactionLogImage.getType()))
-                .transparent(redactionLogImage.isImageHasTransparency())
-                .page(document.getPages()
-                              .stream()
-                              .filter(p -> p.getNumber() == redactionLogImage.getPositions()
-                                      .get(0).getPage())
-                              .findFirst()
-                              .orElseThrow())
-                .position(toRectangle2D(redactionLogImage.getPositions()
-                                                .get(0)))
-                .build();
-
-        List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
-        image.setTreeId(treeId);
-        image.setId(IdBuilder.buildId(image.getPages(),
-                                      image.getBBox().values()
-                                              .stream()
-                                              .toList(),
-                                      "",
-                                      ""));
-
-        return image;
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rectangle rect) {
-
-        return new Rectangle2D.Double(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
-    }
-
-
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
-                .toList();
-
-        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
-                .map(MigrationEntity::getPrecursorEntity)
-                .toList();
-
-        log.info("Finding all possible entities");
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
-
-        for (MigrationEntity migrationEntity : entitiesToMigrate) {
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
-                                                                                                                     tempEntitiesByValue,
-                                                                                                                     MATCH_THRESHOLD);
-
-            if (optionalTextEntity.isEmpty()) {
-                migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
-                migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-                migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
-                continue;
-            }
-
-            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
-
-            migrationEntity.setMigratedEntity(migratedEntity);
-            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-            migrationEntity.setNewId(migratedEntity.getId());
-        }
-
-        tempEntitiesByValue.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .forEach(TextEntity::removeFromGraph);
-
-        return entitiesToMigrate;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
-
-@Builder
-@Getter
-@Setter
-public class ClosestEntity {
-
-    private double distance;
-    private TextEntity textEntity;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import org.kie.api.runtime.KieContainer;
-
-public record KieWrapper(KieContainer container, long rulesVersion) {
-
-    public static KieWrapper empty() {
-
-        return new KieWrapper(null, -1);
-    }
-
-
-    public boolean isPresent() {
-
-        return container != null && rulesVersion >= 0;
-    }
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import java.util.List;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigratedEntityLog {
-
-    MigratedIds migratedIds;
-    EntityLog entityLog;
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -1,441 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@AllArgsConstructor
-@RequiredArgsConstructor
-public final class MigrationEntity {
-
-    private final PrecursorEntity precursorEntity;
-    private final RedactionLogEntry redactionLogEntry;
-    private IEntity migratedEntity;
-    private String oldId;
-    private String newId;
-    private String fileId;
-
-    @Builder.Default
-    List<BaseAnnotation> manualChanges = new LinkedList<>();
-
-
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
-
-        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
-
-        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (redactionLogEntry.lastChangeIsRemoved()) {
-            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied()) {
-            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
-        } else {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        }
-
-        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
-    }
-
-
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
-
-        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
-    }
-
-
-    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getManualChanges() == null) {
-            return false;
-        }
-
-        return redactionLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
-                .orElse(false);
-    }
-
-
-    public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromRedactionLogRectangle)
-                .toList();
-        EntityType entityType = getEntityType(redactionLogEntry, hint);
-        return PrecursorEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(Optional.ofNullable(redactionLogEntry.getReason())
-                                .orElse(""))
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection())
-                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
-                .entityType(entityType)
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .build();
-    }
-
-
-    private static String buildRuleIdentifier(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier;
-        if (redactionLogEntry.getMatchedRule() != null) {
-            ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
-        } else {
-            ruleIdentifier = "MAN.5.0"; // pure ManualRedactions used to have no matched rule
-        }
-        return ruleIdentifier;
-    }
-
-
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        if (hint) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isFalsePositive()) {
-            return EntityType.FALSE_POSITIVE;
-        }
-        if (redactionLogEntry.isHint()) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
-        }
-        return EntityType.ENTITY;
-    }
-
-
-    private static EntryType getEntryType(EntityType entityType) {
-
-        return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
-            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
-            case RECOMMENDATION -> EntryType.RECOMMENDATION;
-            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
-        };
-    }
-
-
-    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
-
-        EntityLogEntry entityLogEntry;
-        if (migratedEntity instanceof Image image) {
-            entityLogEntry = createEntityLogEntry(image);
-        } else if (migratedEntity instanceof TextEntity textEntity) {
-            entityLogEntry = createEntityLogEntry(textEntity);
-        } else if (migratedEntity instanceof PrecursorEntity entity) {
-            entityLogEntry = createEntityLogEntry(entity);
-        } else {
-            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
-        }
-
-        entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
-        entityLogEntry.setColor(redactionLogEntry.getColor());
-        entityLogEntry.setChanges(redactionLogEntry.getChanges()
-                                          .stream()
-                                          .map(MigrationMapper::toEntityLogChanges)
-                                          .toList());
-        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
-        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
-        if (redactionLogEntry.getLegalBasis() != null) {
-            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
-        }
-
-        if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
-            entityLogEntry.setState(EntryState.IGNORED);
-        }
-
-        return entityLogEntry;
-    }
-
-
-    private static boolean lastManualChangeIsRemoveLocally(EntityLogEntry entityLogEntry) {
-
-        return entityLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
-                .isPresent();
-    }
-
-
-    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
-
-        if (ids == null) {
-            return Collections.emptySet();
-        }
-        return ids.stream()
-                .map(oldToNewIdMapping::get)
-                .collect(Collectors.toSet());
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(Image image) {
-
-        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
-        return EntityLogEntry.builder()
-                .id(image.getId())
-                .value(image.value())
-                .type(image.type())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false)
-                .positions(positions)
-                .containingNodeId(image.getTreeId())
-                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .imageHasTransparency(image.isTransparent())
-                .state(buildEntryState(image))
-                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
-                .build();
-
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
-
-        return EntityLogEntry.builder()
-                .id(precursorEntity.getId())
-                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.legalBasis())
-                .value(precursorEntity.value())
-                .type(precursorEntity.type())
-                .state(buildEntryState(precursorEntity))
-                .entryType(buildEntryType(precursorEntity))
-                .section(precursorEntity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .containingNodeId(Collections.emptyList())
-                .closestHeadline("")
-                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(precursorEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite().getPositions()
-                                   .orElse(precursorEntity.getEntityPosition())
-                                   .stream()
-                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                                   .toList())
-                .engines(Collections.emptySet())
-                .build();
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
-
-        assert entity.getPositionsOnPagePerPage().size() == 1;
-        List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
-        return EntityLogEntry.builder()
-                .id(entity.getId())
-                .positions(rectanglesPerLine)
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue()
-                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.type())
-                .section(entity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
-                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(entity.isDictionaryEntry())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end())
-                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
-                .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity))
-                .build();
-    }
-
-
-    private static List<Position> getRectanglesPerLine(TextEntity entity) {
-
-        return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
-                                                               .get(0).getRectanglePerLine()
-                                                               .stream()
-                                                               .map(rectangle2D -> new Position(rectangle2D,
-                                                                                                entity.getPositionsOnPagePerPage()
-                                                                                                        .get(0).getPage().getNumber()))
-                                                               .toList());
-    }
-
-
-    private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
-
-        return entity.getManualOverwrite().getPositions()
-                .map(rects -> rects.stream()
-                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
-                        .toList());
-    }
-
-
-    public boolean hasManualChangesOrComments() {
-
-        return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges();
-    }
-
-
-    public boolean hasManualChanges() {
-
-        return !manualChanges.isEmpty();
-    }
-
-
-    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
-
-        manualChanges.addAll(manualChangesToApply);
-        manualChangesToApply.forEach(manualChange -> {
-            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
-                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
-                // Instead, we add an add_locally change to the db.
-                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
-                        .positions(manualResizeRedaction.getPositions())
-                        .annotationId(getNewId())
-                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
-                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
-                        .textAfter(manualResizeRedaction.getTextAfter())
-                        .textBefore(manualResizeRedaction.getTextBefore())
-                        .build();
-                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
-            } else {
-                migratedEntity.getManualOverwrite().addChange(manualChange);
-            }
-        });
-    }
-
-
-    public ManualRedactionEntry buildManualRedactionEntry() {
-
-        assert hasManualChanges();
-
-        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
-        String user = manualChanges.stream()
-                .filter(mc -> mc instanceof ManualResizeRedaction)
-                .findFirst()
-                .orElse(manualChanges.get(0)).getUser();
-
-        return ManualRedactionEntry.builder()
-                .annotationId(newId)
-                .fileId(fileId)
-                .type(redactionLogEntry.getType())
-                .value(redactionLogEntry.getValue())
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .section(redactionLogEntry.getSection())
-                .addToDictionary(false)
-                .addToDossierDictionary(false)
-                .rectangle(false)
-                .positions(buildPositions(migratedEntity))
-                .user(user)
-                .build();
-    }
-
-
-    private List<Rectangle> buildPositions(IEntity entity) {
-
-        if (entity instanceof TextEntity textEntity) {
-
-            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
-                    .get(0);
-            return positionsOnPage.getRectanglePerLine()
-                    .stream()
-                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
-                    .toList();
-        }
-        if (entity instanceof PrecursorEntity pEntity) {
-
-            return pEntity.getManualOverwrite().getPositions()
-                    .orElse(pEntity.getEntityPosition())
-                    .stream()
-                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
-                                            (float) p.rectangle2D().getY(),
-                                            (float) p.rectangle2D().getWidth(),
-                                            (float) p.rectangle2D().getHeight(),
-                                            p.pageNumber()))
-                    .toList();
-        }
-        if (entity instanceof Image image) {
-
-            Rectangle2D position = image.getManualOverwrite().getPositions()
-                    .map(p -> p.get(0).rectangle2D())
-                    .orElse(image.getPosition());
-
-            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
-
-        } else {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-
-    public boolean needsManualEntry() {
-
-        return manualChanges.stream()
-                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -1,195 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.UUID;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
-public class PrecursorEntity implements IEntity {
-
-    // The id must be mapped into a TextEntity as is for comments to work correctly
-    String id;
-    String value;
-    List<RectangleWithPage> entityPosition;
-    String ruleIdentifier;
-    String reason;
-    String legalBasis;
-    String type;
-    String section;
-    EntityType entityType;
-    boolean applied;
-    boolean isDictionaryEntry;
-    boolean isDossierDictionaryEntry;
-    boolean rectangle;
-    Set<Engine> engines;
-
-    @Builder.Default
-    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    ManualChangeOverwrite manualOverwrite;
-
-
-    public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
-
-        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromAnnotationRectangle)
-                .toList();
-
-        var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
-        ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
-        manualChangeOverwrite.addChange(manualRedactionEntry);
-        return PrecursorEntity.builder()
-                .id(manualRedactionEntry.getAnnotationId())
-                .value(manualRedactionEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier("MAN.0.0")
-                .reason(manualRedactionEntry.getReason())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .type(manualRedactionEntry.getType())
-                .section(manualRedactionEntry.getSection())
-                .entityType(entityType)
-                .applied(true)
-                .isDictionaryEntry(false)
-                .isDossierDictionaryEntry(false)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualOverwrite(manualChangeOverwrite)
-                .engines(Set.of(Engine.MANUAL))
-                .build();
-    }
-
-
-    public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
-
-        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromEntityLogPosition)
-                .toList();
-        EntityType entityType = getEntityType(entityLogEntry.getEntryType());
-        return PrecursorEntity.builder()
-                .id(entityLogEntry.getId())
-                .value(entityLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(entityLogEntry.getMatchedRule())
-                .reason(entityLogEntry.getReason())
-                .legalBasis(entityLogEntry.getLegalBasis())
-                .type(entityLogEntry.getType())
-                .section(entityLogEntry.getSection())
-                .entityType(entityType)
-                .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .engines(entityLogEntry.getEngines())
-                .build();
-    }
-
-
-    public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
-
-        List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromEntityLogPosition)
-                .toList();
-        EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType())
-                .orElse(EntryType.ENTITY);
-        EntityType entityType = getEntityType(entryType);
-        String value = Optional.ofNullable(importedRedaction.getValue())
-                .orElse("");
-        return PrecursorEntity.builder()
-                .id(importedRedaction.getId())
-                .value(value)
-                .entityPosition(rectangleWithPages)
-                .reason(Optional.ofNullable(importedRedaction.getReason())
-                                .orElse(""))
-                .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
-                                    .orElse(""))
-                .type(Optional.ofNullable(importedRedaction.getType())
-                              .orElse(IMPORTED_REDACTION_TYPE))
-                .entityType(entityType)
-                .isDictionaryEntry(false)
-                .isDossierDictionaryEntry(false)
-                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .engines(Set.of(Engine.IMPORTED))
-                .build();
-    }
-
-
-    public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
-
-        List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromAnnotationRectangle)
-                .toList();
-        return PrecursorEntity.builder()
-                .id(UUID.randomUUID().toString())
-                .value(manualResizeRedaction.getValue())
-                .entityPosition(rectangleWithPages)
-                .entityType(EntityType.ENTITY)
-                .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
-                .build();
-    }
-
-
-    @Override
-    public TextRange getTextRange() {
-
-        return new TextRange(-1, -1);
-    }
-
-
-    @Override
-    public String type() {
-
-        return getManualOverwrite().getType()
-                .orElse(type);
-    }
-
-
-    private static EntityType getEntityType(EntryType entryType) {
-
-        switch (entryType) {
-            case FALSE_RECOMMENDATION -> {
-                return EntityType.FALSE_RECOMMENDATION;
-            }
-            case FALSE_POSITIVE -> {
-                return EntityType.FALSE_POSITIVE;
-            }
-            case HINT -> {
-                return EntityType.HINT;
-            }
-            case RECOMMENDATION -> {
-                return EntityType.RECOMMENDATION;
-            }
-            default -> {
-                return EntityType.ENTITY;
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java

@@ -1,45 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import java.awt.geom.Rectangle2D;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-
-public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
-
-    public static RectangleWithPage fromRedactionLogRectangle(Rectangle rectangle) {
-
-        return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
-    }
-
-
-    public static RectangleWithPage fromEntityLogPosition(Position position) {
-
-        return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position));
-    }
-
-
-    public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
-
-        return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rectangle rectangle) {
-
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
-    }
-
-
-    private static Rectangle2D toRectangle2D(Position position) {
-
-        return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h());
-    }
-
-
-    private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
-
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java

@@ -1,41 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.component;
-
-import java.util.Collection;
-import java.util.List;
-
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Component {
-
-    RuleIdentifier matchedRule;
-    String name;
-    String value;
-    String valueDescription;
-
-    List<Entity> references;
-
-
-    public boolean addReference(Entity entity) {
-
-        return references.add(entity);
-    }
-
-
-    public boolean addReferences(Collection<Entity> entities) {
-
-        return references.addAll(entities);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -1,104 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.component;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class Entity {
-
-    String id;
-    String type;
-    EntryType entryType;
-    EntryState state;
-    String value;
-    String reason;
-    String matchedRule;
-    String legalBasis;
-
-    boolean imported;
-
-    SemanticNode containingNode;
-    String section;
-    float[] color;
-
-    List<Position> positions;
-
-    String textBefore;
-    String textAfter;
-
-    int startOffset;
-    int endOffset;
-    int length;
-
-    boolean imageHasTransparency;
-
-    boolean isDictionaryEntry;
-    boolean isDossierDictionaryEntry;
-
-    boolean excluded;
-
-    List<Change> changes;
-
-    List<ManualChange> manualChanges;
-
-    Set<Engine> engines;
-
-    Set<String> reference;
-
-    Set<String> importedRedactionIntersections;
-
-
-    public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
-
-        return Entity.builder()
-                .id(e.getId())
-                .type(e.getType())
-                .entryType(e.getEntryType())
-                .state(e.getState())
-                .value(e.getValue())
-                .reason(e.getReason())
-                .matchedRule(e.getMatchedRule())
-                .legalBasis(e.getLegalBasis())
-                .imported(e.isImported())
-                .section(e.getSection())
-                .color(e.getColor())
-                .positions(e.getPositions())
-                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
-                .textBefore(e.getTextBefore())
-                .textAfter(e.getTextAfter())
-                .startOffset(e.getStartOffset())
-                .endOffset(e.getEndOffset())
-                .length(Optional.ofNullable(e.getValue()).orElse("").length())
-                .imageHasTransparency(e.isImageHasTransparency())
-                .isDictionaryEntry(e.isDictionaryEntry())
-                .isDossierDictionaryEntry(e.isDossierDictionaryEntry())
-                .excluded(e.isExcluded())
-                .changes(e.getChanges())
-                .manualChanges(e.getManualChanges())
-                .engines(e.getEngines())
-                .reference(e.getReference())
-                .importedRedactionIntersections(e.getImportedRedactionIntersections())
-                .build();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -1,251 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.experimental.FieldDefaults;
-
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ManualChangeOverwrite {
-
-    private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-                                                                                                          ManualRedactionEntry.class, "created by manual change", //
-                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
-                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
-                                                                                                          ManualForceRedaction.class, "forced by manual override", //
-                                                                                                          IdRemoval.class, "removed by manual override", //
-                                                                                                          ManualRecategorization.class, "recategorized by manual override");
-
-    List<BaseAnnotation> manualChanges = new LinkedList<>();
-    boolean changed;
-    List<String> descriptions;
-    String type;
-    String legalBasis;
-    String section;
-    String value;
-    List<RectangleWithPage> positions;
-    Boolean applied;
-    Boolean removed;
-    Boolean ignored;
-    Boolean resized;
-    Boolean recategorized;
-    EntityType entityType;
-
-
-    public ManualChangeOverwrite() {
-
-        this(EntityType.ENTITY);
-    }
-
-
-    public ManualChangeOverwrite(EntityType entityType) {
-
-        this.entityType = entityType;
-    }
-
-
-    public void calculateCurrentOverride() {
-
-        if (!changed) {
-            return;
-        }
-        List<BaseAnnotation> sortedManualChanges = getManualChangeLog();
-        updateFields(sortedManualChanges);
-    }
-
-
-    public List<BaseAnnotation> getManualChangeLog() {
-
-        if (!changed) {
-            return manualChanges;
-        }
-        manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
-        updateFields(manualChanges);
-        // make list unmodifiable.
-        return manualChanges.stream()
-                .toList();
-    }
-
-
-    private void updateFields(List<BaseAnnotation> sortedManualChanges) {
-
-        descriptions = new LinkedList<>();
-
-        for (BaseAnnotation manualChange : sortedManualChanges) {
-            // ManualRedactionEntries are created prior to rule execution in analysis service.
-
-            if (manualChange instanceof IdRemoval) {
-                applied = false;
-                if (sortedManualChanges.get(0) instanceof ManualRedactionEntry) { // set to removed if it's a Manual Redaction
-                    removed = true;
-                } else {
-                    ignored = true;
-                }
-            }
-
-            if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) {
-                setToApplied();
-                legalBasis = manualRedactionEntry.getLegalBasis();
-            }
-
-            if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
-                setToApplied();
-                legalBasis = manualForceRedaction.getLegalBasis();
-            }
-
-            if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                section = manualLegalBasisChange.getSection();
-                legalBasis = manualLegalBasisChange.getLegalBasis();
-                value = manualLegalBasisChange.getValue();
-            }
-
-            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
-                // resizing logic for TextEntities and Images happens in ManualChangesApplicationService.
-                resized = true;
-                // This is only for not found Manual Entities.
-                value = manualResizeRedaction.getValue();
-                positions = manualResizeRedaction.getPositions()
-                        .stream()
-                        .map(RectangleWithPage::fromAnnotationRectangle)
-                        .toList();
-            }
-
-            if (manualChange instanceof ManualRecategorization recategorization) {
-                recategorized = true;
-                type = recategorization.getType();
-                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
-                    legalBasis = recategorization.getLegalBasis();
-                }
-            }
-
-            descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
-        }
-        changed = false;
-    }
-
-
-    private void setToApplied() {
-
-        removed = false;
-        ignored = false;
-        if (!isHint(entityType)) {
-            applied = true;
-        }
-    }
-
-
-    public void addChange(BaseAnnotation manualChange) {
-
-        changed = true;
-        manualChanges.add(manualChange);
-    }
-
-
-    public void addChanges(List<BaseAnnotation> manualChangeLog) {
-
-        changed = true;
-        manualChanges.addAll(manualChangeLog);
-    }
-
-
-    public Optional<String> getLegalBasis() {
-
-        calculateCurrentOverride();
-        return legalBasis == null ? Optional.empty() : Optional.of(legalBasis);
-    }
-
-
-    public Optional<String> getType() {
-
-        calculateCurrentOverride();
-        return type == null ? Optional.empty() : Optional.of(type);
-    }
-
-
-    public Optional<String> getSection() {
-
-        calculateCurrentOverride();
-        return section == null ? Optional.empty() : Optional.of(section);
-    }
-
-
-    public Optional<String> getValue() {
-
-        calculateCurrentOverride();
-        return value == null ? Optional.empty() : Optional.of(value);
-    }
-
-
-    public Optional<Boolean> getApplied() {
-
-        calculateCurrentOverride();
-        return applied == null ? Optional.empty() : Optional.of(applied);
-    }
-
-
-    public Optional<Boolean> getRemoved() {
-
-        calculateCurrentOverride();
-        return removed == null ? Optional.empty() : Optional.of(removed);
-    }
-
-
-    public Optional<Boolean> getIgnored() {
-
-        calculateCurrentOverride();
-        return ignored == null ? Optional.empty() : Optional.of(ignored);
-    }
-
-
-    public Optional<Boolean> getResized() {
-
-        calculateCurrentOverride();
-        return resized == null ? Optional.empty() : Optional.of(resized);
-    }
-
-
-    public Optional<Boolean> getRecategorized() {
-
-        calculateCurrentOverride();
-        return recategorized == null ? Optional.empty() : Optional.of(recategorized);
-    }
-
-
-    public List<String> getDescriptions() {
-
-        calculateCurrentOverride();
-        return descriptions == null ? Collections.emptyList() : descriptions;
-    }
-
-
-    public Optional<List<RectangleWithPage>> getPositions() {
-
-        calculateCurrentOverride();
-        return positions == null ? Optional.empty() : Optional.of(positions);
-    }
-
-
-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -1,103 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleType;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Builder
-@AllArgsConstructor
-@EqualsAndHashCode
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public final class MatchedRule implements Comparable<MatchedRule> {
-
-    public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
-    public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
-    public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
-
-    RuleIdentifier ruleIdentifier;
-    @Builder.Default
-    String reason = "";
-    @Builder.Default
-    String legalBasis = "";
-    boolean writeValueWithLineBreaks;
-    boolean applied;
-    boolean removed;
-    boolean ignored;
-
-    @Builder.Default
-    Set<TextEntity> references = Collections.emptySet();
-
-
-    public static MatchedRule empty() {
-
-        return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
-    }
-
-
-    public MatchedRule asSkippedIfApplied() {
-
-        if (!this.isApplied()) {
-            return this;
-        }
-        return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
-                .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
-                .legalBasis(this.getLegalBasis())
-                .reason(this.getReason())
-                .references(this.getReferences())
-                .build();
-    }
-
-
-    @Override
-    public int compareTo(MatchedRule matchedRule) {
-
-        // Only the highest ranked rule is actually applied, this method defines the highest order.
-        // First, it compares the Rule Type, RULE_TYPE_PRIORITIES defines the order of types.
-        // Types not in the list have the lowest priority.
-        // The ones in the list are technical exceptions and should override any other Rule.
-        // Aside from them Entities should never match from more than one type!
-        // E.g. a CBI_author entity should **always** only match CBI.*.* rules.
-        // Otherwise, something went wrong with the rules. :)
-        RuleIdentifier otherRuleIdentifier = matchedRule.getRuleIdentifier();
-        boolean thisInList = RULE_TYPE_PRIORITIES.contains(this.getRuleIdentifier().type());
-        boolean otherInList = RULE_TYPE_PRIORITIES.contains(otherRuleIdentifier.type());
-
-        // Compare the types
-        if (thisInList && !otherInList) {
-            return -1;
-        } else if (!thisInList && otherInList) {
-            return 1;
-        } else if (thisInList && otherInList) {
-            int thisIndex = RULE_TYPE_PRIORITIES.indexOf(this.getRuleIdentifier().type());
-            int otherIndex = RULE_TYPE_PRIORITIES.indexOf(otherRuleIdentifier.type());
-            return Integer.compare(thisIndex, otherIndex);
-        }
-        // Then compare the unit
-        if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) {
-            return Integer.compare(otherRuleIdentifier.unit(), ruleIdentifier.unit());
-        }
-        // Then compare the id inside the unit
-        return Integer.compare(otherRuleIdentifier.id(), ruleIdentifier.id());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return "MatchedRule[ruleIdentifier=" + ruleIdentifier + ", reason=" + reason + ", legalBasis=" + legalBasis + ", applied=" + applied + ", writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", references=" + references + ']';
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -1,237 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.document.entity;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Set;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
-public class TextEntity implements IEntity {
-
-    // primary key
-    @EqualsAndHashCode.Include
-    final String id;
-    // primary key end
-
-    TextRange textRange;
-    @Builder.Default
-    List<TextRange> duplicateTextRanges = new ArrayList<>();
-    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
-    final EntityType entityType;
-
-    @Builder.Default
-    final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    final ManualChangeOverwrite manualOverwrite;
-
-    boolean dictionaryEntry;
-    boolean dossierDictionaryEntry;
-
-    @Builder.Default
-    Set<Engine> engines = new HashSet<>();
-
-    // inferred on graph insertion
-    String value;
-    String textBefore;
-    String textAfter;
-    @Builder.Default
-    Set<Page> pages = new HashSet<>();
-    List<PositionOnPage> positionsOnPagePerPage;
-    @Builder.Default
-    List<SemanticNode> intersectingNodes = new LinkedList<>();
-    SemanticNode deepestFullyContainingNode;
-
-
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
-
-        return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
-    }
-
-
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
-
-        return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
-    }
-
-
-    private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
-
-        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
-        return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
-    }
-
-
-    public void addTextRange(TextRange textRange) {
-
-        duplicateTextRanges.add(textRange);
-    }
-
-    public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
-
-        return intersectingNodes.stream().anyMatch(clazz::isInstance);
-    }
-
-
-    public boolean occursInNode(SemanticNode semanticNode) {
-
-        return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode));
-    }
-
-
-    public boolean isType(String type) {
-
-        return type().equals(type);
-    }
-
-
-    public boolean isAnyType(List<String> types) {
-
-        return types.contains(type());
-    }
-
-
-    public void addIntersectingNode(SemanticNode containingNode) {
-
-        intersectingNodes.add(containingNode);
-    }
-
-
-    public String getValueWithLineBreaks() {
-
-        return getDeepestFullyContainingNode().getTextBlock().subSequenceWithLineBreaks(getTextRange());
-    }
-
-
-    public void removeFromGraph() {
-
-        intersectingNodes.forEach(node -> node.getEntities().remove(this));
-        pages.forEach(page -> page.getEntities().remove(this));
-        intersectingNodes = new LinkedList<>();
-        deepestFullyContainingNode = null;
-        pages = new HashSet<>();
-        remove("FINAL.0.0", "removed completely");
-    }
-
-
-    public List<PositionOnPage> getPositionsOnPagePerPage() {
-
-        if (positionsOnPagePerPage == null || positionsOnPagePerPage.isEmpty()) {
-            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(textRange);
-
-            Page firstPage = rectanglesPerLinePerPage.keySet()
-                    .stream()
-                    .min(Comparator.comparingInt(Page::getNumber))
-                    .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
-
-            positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
-        }
-        return positionsOnPagePerPage;
-    }
-
-
-    private static PositionOnPage buildPositionOnPage(Page firstPage, String id, Map.Entry<Page, List<Rectangle2D>> entry) {
-
-        if (entry.getKey().equals(firstPage)) {
-            return new PositionOnPage(id, entry.getKey(), entry.getValue());
-        } else {
-            return new PositionOnPage(id + "-" + entry.getKey().getNumber(), entry.getKey(), entry.getValue());
-        }
-    }
-
-
-    public boolean containedBy(TextEntity textEntity) {
-
-        return this.textRange.containedBy(textEntity.getTextRange());
-    }
-
-
-    public boolean contains(TextEntity textEntity) {
-
-        return this.textRange.contains(textEntity.getTextRange());
-    }
-
-
-    public boolean intersects(TextEntity textEntity) {
-
-        return this.textRange.intersects(textEntity.getTextRange());
-    }
-
-
-    public void addEngine(Engine engine) {
-
-        engines.add(engine);
-    }
-
-
-    public void addEngines(Set<Engine> engines) {
-
-        this.engines.addAll(engines);
-    }
-
-
-    public boolean matchesAnnotationId(String manualRedactionId) {
-
-        return getPositionsOnPagePerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
-    }
-
-
-    @Override
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("Entity[\"");
-        sb.append(value);
-        sb.append("\", ");
-        sb.append(textRange);
-        sb.append(", pages[");
-        pages.forEach(page -> {
-            sb.append(page.getNumber());
-            sb.append(", ");
-        });
-        sb.delete(sb.length() - 2, sb.length());
-        sb.append("], type = \"");
-        sb.append(type());
-        sb.append("\", EntityType.");
-        sb.append(entityType);
-        sb.append("]");
-        return sb.toString();
-    }
-
-
-    @Override
-    public String type() {
-
-        return getManualOverwrite().getType().orElse(type);
-    }
-
-
-    @Override
-    public String value() {
-
-        return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -1,165 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.document.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-public class Image implements GenericSemanticNode, IEntity {
-
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
-    List<Integer> treeId;
-    String id;
-
-    ImageType imageType;
-    boolean transparent;
-    Rectangle2D position;
-
-    @Builder.Default
-    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-
-    @Builder.Default
-    ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
-
-    @EqualsAndHashCode.Exclude
-    Page page;
-
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<TextEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.IMAGE;
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return streamAllSubNodes().filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
-                .collect(new TextBlockCollector());
-    }
-
-
-    @Override
-    public Set<Page> getPages() {
-
-        return Collections.singleton(page);
-    }
-
-
-    @Override
-    public TextRange getTextRange() {
-
-        return GenericSemanticNode.super.getTextRange();
-    }
-
-
-    @Override
-    public String type() {
-
-        return getManualOverwrite().getType()
-                .orElse(imageType.toString());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
-    }
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        bBoxPerPage.put(page, position);
-        return bBoxPerPage;
-    }
-
-
-    @Override
-    public String getValue() {
-
-        return NodeType.IMAGE + ":" + camelCase(imageType.toString());
-    }
-
-
-    private String camelCase(String name) {
-
-        return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
-    }
-
-
-    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
-
-        Map<Page, Rectangle2D> bboxImage = image.getBBox();
-        Map<Page, Rectangle2D> bbox = this.getBBox();
-        //image needs to be on the same page
-        if(bboxImage.get(this.page) != null) {
-            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
-            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
-            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
-            return (calculatedIntersection / area) > containmentThreshold;
-        }
-        return false;
-    }
-
-
-    public boolean mostlyContains(Image image, double containmentThreshold) {
-
-        Map<Page, Rectangle2D> bboxImage = image.getBBox();
-        Map<Page, Rectangle2D> bbox = this.getBBox();
-        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
-        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
-        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
-        return (area / calculatedIntersection) > containmentThreshold;
-    }
-
-
-    public int length() {
-
-        return 0;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicQuery.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import lombok.AccessLevel;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class BasicQuery {
-
-    String name;
-    int line;
-    String code;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java

@@ -1,42 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import org.drools.drl.ast.descr.AttributeDescr;
-import org.drools.drl.ast.descr.RuleDescr;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@AllArgsConstructor
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class BasicRule {
-
-    @EqualsAndHashCode.Include
-    RuleIdentifier identifier;
-    String name;
-    String code;
-    String agendaGroup;
-    int line;
-
-
-    public static BasicRule fromRuleDescr(RuleDescr rule, String rulesString) {
-
-        RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
-        String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
-        String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
-        String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
-        return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return "BasicRule[identifier=" + identifier + ", name=" + name + ", code=" + code + ']';
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-
-public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
-
-    public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
-
-        return ruleUnits.stream()
-                .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -1,84 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public final class RuleFileBluePrint {
-
-    String imports;
-    int importLine;
-    String globals;
-    int globalsLine;
-    List<BasicQuery> queries;
-    List<RuleClass> ruleClasses;
-    DroolsSyntaxValidation droolsSyntaxValidation;
-
-
-    public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
-
-        return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
-    }
-
-
-    public Set<String> getImportSplitByKeyword() {
-
-        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
-    }
-
-
-    public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
-
-        if (Objects.isNull(ruleIdentifier.unit())) {
-            return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits)
-                    .orElse(Collections.emptyList())
-                    .stream()
-                    .flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
-                    .toList();
-        }
-        return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()))
-                .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
-                .orElse(Stream.empty())
-                .toList();
-    }
-
-
-    public List<RuleIdentifier> getAllRuleIdentifiers() {
-
-        return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList());
-    }
-
-
-    public Stream<BasicRule> streamAllRules() {
-
-        return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']';
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java

@@ -1,81 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import java.util.Objects;
-
-import lombok.NonNull;
-
-public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
-
-    public static RuleIdentifier fromName(String name) {
-
-        String identifier = name.split(":")[0];
-        return fromString(identifier);
-    }
-
-
-    public static RuleIdentifier fromString(String identifier) {
-
-        String[] values = identifier.split("\\.");
-        if (values.length != 3) {
-            throw new IllegalArgumentException(String.format("Identifier %s must conform to the pattern \"[a-zA-Z0-9]+.\\d+.\\d+\"", identifier));
-        }
-        RuleType type = RuleType.fromString(values[0]);
-        Integer group = Integer.parseInt(values[1]);
-        Integer id = Integer.parseInt(values[2]);
-        return new RuleIdentifier(type, group, id);
-    }
-
-
-    public static RuleIdentifier empty() {
-
-        return new RuleIdentifier(new RuleType(""), null, null);
-    }
-
-
-    /**
-     * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
-     * Therefore, to compare RuleIdentifiers one should always use the function matches.
-     *
-     * @param ruleIdentifier RuleIdentifier to check if this one matches it.
-     * @return true, if all fields match. If a field is null, it matches any field.
-     */
-    public boolean matches(RuleIdentifier ruleIdentifier) {
-
-        return ruleIdentifier.type().equals(this.type()) && //
-                (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
-                (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
-
-    }
-
-
-    @Override
-    public String toString() {
-
-        if (type.name().isBlank() || type.name().isEmpty()) {
-            return "";
-        }
-
-        StringBuilder sb = new StringBuilder();
-        sb.append(type().name());
-        if (Objects.nonNull(unit())) {
-            sb.append(".");
-            sb.append(unit());
-        } else {
-            sb.append(".*");
-        }
-        if (Objects.nonNull(id())) {
-            sb.append(".");
-            sb.append(id());
-        } else {
-            sb.append(".*");
-        }
-        return sb.toString();
-    }
-
-
-    public String toRuleUnitString() {
-
-        return String.format("Rule unit: %s.%d", type.name(), unit);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleType.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import java.util.regex.Pattern;
-
-public record RuleType(String name) {
-
-    static Pattern alphaNumeric = Pattern.compile("^[a-zA-Z0-9]+$");
-
-
-    public static RuleType fromString(String value) {
-
-        if (!alphaNumeric.matcher(value).matches()) {
-            throw new IllegalArgumentException(String.format("The rule type %s must only contain numbers and letters!", value));
-        }
-        return new RuleType(value);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleUnit.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import java.util.List;
-
-public record RuleUnit(Integer unit, List<BasicRule> rules) {
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,11 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import lombok.RequiredArgsConstructor;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
@@ -13,12 +8,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import lombok.RequiredArgsConstructor;
-
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
+    public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_DQL = "redactionDQL";
+
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
@@ -39,20 +37,6 @@ public class MessagingConfiguration {
     }
 
 
-    @Bean
-    public Queue migrationQueue() {
-
-        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue migrationDLQ() {
-
-        return QueueBuilder.durable(MIGRATION_DLQ).build();
-    }
-
-
     @Bean
     public Queue redactionQueue() {
 
@@ -70,16 +54,6 @@ public class MessagingConfiguration {
                 .build();
     }
 
-    @Bean
-    public Queue redactionAnalysisResponseQueue() {
-
-        return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
 
     @Bean
     public Queue redactionDeadLetterQueue() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -22,10 +22,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -39,8 +37,7 @@ public class RedactionMessageReceiver {
     private final ObjectMapper objectMapper;
     private final AnalyzeService analyzeService;
     private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
-    private final RulesClient rulesClient;
-    private final UnprocessedChangesService unprocessedChangesService;
+    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
 
 
     @SneakyThrows
@@ -72,8 +69,7 @@ public class RedactionMessageReceiver {
         log.info("");
         log.info("Processing AnalyzeRequest for file: {},  priority: {} ", analyzeRequest.getFileId(), priority);
         log.info("");
-        AnalyzeResult result = new AnalyzeResult();
-        boolean shouldRespond = true;
+        AnalyzeResult result;
 
         try {
             switch (analyzeRequest.getMessageType()) {
@@ -101,50 +97,40 @@ public class RedactionMessageReceiver {
                             format("%.2f", result.getDuration() / 1000.0));
                     log.info("----------------------------------------------------------------------------------");
                     break;
-                case SURROUNDING_TEXT_ANALYSIS:
-                    log.info("------------------------------Surrounding Text Analysis------------------------------------------");
-                    log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-                    log.debug(analyzeRequest.getManualRedactions().toString());
-                    unprocessedChangesService.analyseSurroundingText(analyzeRequest);
-                    log.info("Successful Surrounding Text Analysis dossier {} file {} ",
+
+                case SURROUNDING_TEXT:
+                    log.info("------------------------------Add surrounding Text--------------------------------");
+                    log.info("Starting surrounding text search for entries {} ", analyzeRequest.getManualRedactions().getEntriesToAdd());
+                    result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(),
+                            analyzeRequest.getFileId(),
+                            analyzeRequest.getManualRedactions());
+                    log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {} s",
                             analyzeRequest.getDossierId(),
-                            analyzeRequest.getFileId());
-                    log.info("-------------------------------------------------------------------------------------------------");
-                    shouldRespond = false;
+                            analyzeRequest.getFileId(),
+                            format("%.2f", result.getDuration() / 1000.0));
+                    log.info("----------------------------------------------------------------------------------");
                     break;
 
                 default:
                     throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
             }
             log.info("");
-            if (shouldRespond) {
-                result.setMessageType(analyzeRequest.getMessageType());
-                fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
-            }
-        } catch (DroolsTimeoutException droolsTimeoutException) {
-            if (!droolsTimeoutException.isReported()) {
-                rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId(), droolsTimeoutException.getRuleFileType());
-            }
-            sendAnalysisFailed(analyzeRequest, priority, droolsTimeoutException);
+            result.setMessageType(analyzeRequest.getMessageType());
+            fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
+
         } catch (Exception e) {
-            sendAnalysisFailed(analyzeRequest, priority, e);
+            log.warn("Failed to process analyze request: {}", analyzeRequest, e);
+            var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+            fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
+                    analyzeRequest.getFileId(),
+                    new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
         }
     }
 
 
-    private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
-
-        log.warn("Failed to process analyze request: {}", analyzeRequest, e);
-        var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
-        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
-    }
-
-
     @RabbitHandler
     @RabbitListener(queues = REDACTION_DQL)
-    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
+    public void receiveAnalyzeRequestDQL(Message in) throws IOException {
 
         var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/CustomEntityCreationAdapter.java

@@ -0,0 +1,298 @@
+package com.iqser.red.service.redaction.v1.server.redaction.adapter;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class CustomEntityCreationAdapter {
+
+    private static final double MATCH_THRESHOLD = 5; // sum of distances for each corner
+    private final EntityCreationService entityCreationService;
+
+
+    @Autowired
+    public CustomEntityCreationAdapter(EntityEnrichmentService entityEnrichmentService) {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+    }
+
+
+    public List<RedactionEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
+
+        List<EntityIdentifier> entityIdentifiers = redactionLog.getRedactionLogEntry().stream().map(EntityIdentifier::fromRedactionLogEntry).toList();
+        return toRedactionEntity(entityIdentifiers, node);
+    }
+
+
+    public List<RedactionEntity> createRedactionEntities(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) {
+
+        List<EntityIdentifier> entityIdentifiers = manualRedactionEntries.stream()
+                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
+                .map(EntityIdentifier::fromManualRedactionEntry)
+                .toList();
+
+        return toRedactionEntity(entityIdentifiers, node);
+    }
+
+
+    private List<RedactionEntity> toRedactionEntity(List<EntityIdentifier> entityIdentifiers, SemanticNode node) {
+
+        Set<Integer> pageNumbers = entityIdentifiers.stream().flatMap(entry -> entry.entityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
+        Set<String> entryValues = entityIdentifiers.stream().map(EntityIdentifier::value).map(String::toLowerCase).collect(Collectors.toSet());
+
+        Map<String, List<RedactionEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
+        assert allValuesFound(tempEntitiesByValue, entryValues);
+
+        List<RedactionEntity> correctEntities = entityIdentifiers.stream()
+                .map(entityIdentifier -> findClosestEntity(entityIdentifier, tempEntitiesByValue).map(tempEntity -> createCorrectEntity(entityIdentifier,
+                        node,
+                        tempEntity.getBoundary())))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .toList();
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(RedactionEntity::removeFromGraph);
+        return correctEntities;
+    }
+
+
+    /**
+     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
+     *
+     * @param entityIdentifier The entity identifier for the RedactionEntity.
+     * @param node             The SemanticNode associated with the RedactionEntity.
+     * @param closestBoundary  The closest Boundary to the RedactionEntity.
+     * @return The created correct RedactionEntity.
+     */
+    private RedactionEntity createCorrectEntity(EntityIdentifier entityIdentifier, SemanticNode node, Boundary closestBoundary) {
+
+        RedactionEntity correctEntity = entityCreationService.forceByBoundary(closestBoundary, entityIdentifier.type(), entityIdentifier.entityType, node);
+
+        if (entityIdentifier.redacted()) {
+            correctEntity.force(entityIdentifier.ruleIdentifier(), entityIdentifier.reason(), entityIdentifier.legalBasis());
+        } else {
+            correctEntity.skip(entityIdentifier.ruleIdentifier(), entityIdentifier.reason());
+        }
+        correctEntity.setDictionaryEntry(entityIdentifier.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(entityIdentifier.isDossierDictionaryEntry());
+        return correctEntity;
+    }
+
+
+    private Optional<RedactionEntity> findClosestEntity(EntityIdentifier identifier, Map<String, List<RedactionEntity>> entitiesWithSameValue) {
+
+        List<RedactionEntity> possibleEntities = entitiesWithSameValue.get(identifier.value().toLowerCase(Locale.ROOT));
+
+        if (possibleEntities == null || possibleEntities.isEmpty()) {
+            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.value());
+            return Optional.empty();
+        }
+
+        Optional<RedactionEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, identifier.entityPosition()))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.entityPosition(), entity)));
+
+        if (optionalClosestEntity.isEmpty()) {
+            log.warn("No Entity with value {} found on page {}", identifier.value(), identifier.entityPosition());
+            return optionalClosestEntity;
+        }
+
+        RedactionEntity closestEntity = optionalClosestEntity.get();
+        double distance = calculateMinDistance(identifier.entityPosition(), closestEntity);
+        if (distance > MATCH_THRESHOLD) {
+            log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
+                    distance,
+                    MATCH_THRESHOLD,
+                    identifier.entityPosition(),
+                    closestEntity.getRedactionPositionsPerPage()));
+            return Optional.empty();
+        }
+
+        return Optional.of(closestEntity);
+    }
+
+
+    private Map<String, List<RedactionEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
+
+        if (!pageNumbers.stream().allMatch(node::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
+                    node.getPages()));
+        }
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
+                .stream()
+                .map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+
+    private static boolean allValuesFound(Map<String, List<RedactionEntity>> entitiesByValue, Set<String> entryValues) {
+
+        return entitiesByValue.keySet().equals(entryValues);
+    }
+
+
+    private static boolean pagesMatch(RedactionEntity entity, List<RectangleWithPage> originalPositions) {
+
+        Set<Integer> entityPageNumbers = entity.getRedactionPositionsPerPage().stream().map(RedactionPosition::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        return entityPageNumbers.containsAll(originalPageNumbers);
+    }
+
+
+    private double calculateMinDistance(List<RectangleWithPage> originalPositions, RedactionEntity entity) {
+
+        if (originalPositions.size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return originalPositions.stream()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .sum();
+    }
+
+
+    private static long countRectangles(RedactionEntity entity) {
+
+        return entity.getRedactionPositionsPerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(RedactionEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+
+        return entity.getRedactionPositionsPerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
+                .map(RedactionPosition::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
+
+        // mirrored coordinates safe comparison
+        double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
+        double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
+        double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
+        double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
+
+        double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
+        double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
+        double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
+        double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
+
+        return Math.abs(minX1 - minX2) //
+                + Math.abs(minY1 - minY2) //
+                + Math.abs(maxX1 - maxX2) //
+                + Math.abs(maxY1 - maxY2);
+    }
+
+
+    private record EntityIdentifier(
+            String value,
+            List<RectangleWithPage> entityPosition,
+            String ruleIdentifier,
+            String reason,
+            String legalBasis,
+            String type,
+            EntityType entityType,
+            boolean redacted,
+            boolean isDictionaryEntry,
+            boolean isDossierDictionaryEntry) {
+
+        public static EntityIdentifier fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+
+            String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
+            List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            return new EntityIdentifier(redactionLogEntry.getValue(),
+                    rectangleWithPages,
+                    ruleIdentifier,
+                    redactionLogEntry.getReason(),
+                    redactionLogEntry.getLegalBasis(),
+                    redactionLogEntry.getType(),
+                    redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
+                    redactionLogEntry.isRedacted(),
+                    redactionLogEntry.isDictionaryEntry(),
+                    redactionLogEntry.isDossierDictionaryEntry());
+        }
+
+
+        public static EntityIdentifier fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
+
+            List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+            return new EntityIdentifier(manualRedactionEntry.getValue(),
+                    rectangleWithPages,
+                    "MAN.0.0",
+                    manualRedactionEntry.getReason(),
+                    manualRedactionEntry.getLegalBasis(),
+                    manualRedactionEntry.getType(),
+                    EntityType.ENTITY,
+                    true,
+                    false,
+                    false);
+        }
+
+    }
+
+    private record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
+
+        public static RectangleWithPage fromRedactionLogRectangle(Rectangle rectangle) {
+
+            return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
+        }
+
+
+        public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
+
+            return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
+        }
+
+
+        private static Rectangle2D toRectangle2D(Rectangle rectangle) {
+
+            return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        }
+
+
+        private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
+
+            return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java

@@ -1,10 +1,10 @@
-package com.iqser.red.service.redaction.v1.server.model;
+package com.iqser.red.service.redaction.v1.server.redaction.adapter;
 
 import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -37,7 +37,7 @@ public class NerEntities {
     }
 
 
-    public record NerEntity(String value, TextRange textRange, String type) {
+    public record NerEntity(String value, Boundary boundary, String type) {
 
     }