RED-4645: Multitenancy for storage

Merge in RED/redaction-service from RED-6162 to master

* commit '08d20e30e347c79ab4667a524ab814e160778074':
  RED-6162 - bumped version

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -12,7 +12,8 @@ mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
 echo "dependency-check:aggregate"
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    org.owasp:dependency-check-maven:aggregate
+    org.owasp:dependency-check-maven:aggregate \
+    -DknownExploitedEnabled=false
 
 if [[ -z "${bamboo_repository_pr_key}" ]]
 then

pom.xml

@@ -7,7 +7,7 @@
 
     <artifactId>redaction-service</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>3.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
 
     <packaging>pom</packaging>

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.10.0</version>
+        <version>1.17.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.22.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.299.0</persistence-service.version>
+        <persistence-service.version>2.1.0</persistence-service.version>
     </properties>
 
     <dependencies>
@@ -32,13 +32,18 @@
 
         <dependency>
             <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-api-v1</artifactId>
+            <artifactId>persistence-service-internal-api-v1</artifactId>
             <version>${persistence-service.version}</version>
             <exclusions>
+
                 <exclusion>
                     <groupId>com.iqser.red.service</groupId>
                     <artifactId>redaction-service-api-v1</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>persistence-service-api-v1</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java

@@ -1,40 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeRequest {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    private OffsetDateTime lastProcessed;
-    private int analysisNumber;
-
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-    @Builder.Default
-    private Set<Integer> sectionsToReanalyse = new HashSet<>();
-
-    @Builder.Default
-    private List<FileAttribute> fileAttributes = new ArrayList<>();
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java

@@ -1,36 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeResult {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private long duration;
-    private int numberOfPages;
-    private boolean hasUpdates;
-    private long dictionaryVersion;
-    private long dossierDictionaryVersion;
-    private long rulesVersion;
-    private long legalBasisVersion;
-
-    private boolean wasReanalyzed;
-
-    private int analysisVersion;
-    private int analysisNumber;
-
-    private ManualRedactions manualRedactions;
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/CellRectangle.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class CellRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class Change {
-
-    private int analysisNumber;
-    private ChangeType type;
-    private OffsetDateTime dateTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ChangeType {
-    ADDED,
-    REMOVED,
-    CHANGED
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Engine.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum Engine {
-    DICTIONARY,
-    NER,
-    RULE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java

@@ -1,19 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class FileAttribute {
-
-    private String id;
-    private String label;
-    private String placeholder;
-    private String value;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedaction.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedaction {
-
-    private String id;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java

@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@CompiledJson
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedactions {
-
-    @Builder.Default
-    private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java

@@ -1,59 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-public class ManualChange {
-
-    private AnnotationStatus annotationStatus;
-    private ManualRedactionType manualRedactionType;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime requestedDate;
-    private String userId;
-    private Map<String, String> propertyChanges = new HashMap<>();
-
-
-    public static ManualChange from(BaseAnnotation baseAnnotation) {
-
-        ManualChange manualChange = new ManualChange();
-        manualChange.annotationStatus = baseAnnotation.getStatus();
-        manualChange.processedDate = baseAnnotation.getProcessedDate();
-        manualChange.requestedDate = baseAnnotation.getRequestDate();
-        manualChange.userId = baseAnnotation.getUser();
-        return manualChange;
-    }
-
-
-    public boolean isProcessed() {
-
-        return processedDate != null;
-    }
-
-
-    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        this.manualRedactionType = manualRedactionType;
-        return this;
-    }
-
-
-    public ManualChange withChange(String property, String value) {
-
-        this.propertyChanges.put(property, value);
-        return this;
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java

@@ -1,13 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ManualRedactionType {
-    ADD_LOCALLY,
-    ADD_TO_DICTIONARY,
-    REMOVE_LOCALLY,
-    REMOVE_FROM_DICTIONARY,
-    FORCE_REDACT,
-    FORCE_HINT,
-    RECATEGORIZE,
-    LEGAL_BASIS_CHANGE,
-    RESIZE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MessageType.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum MessageType {
-
-    ANALYSE,
-    REANALYSE,
-    STRUCTURE_ANALYSE,
-    SURROUNDING_TEXT
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Point {
-
-    private float x;
-    private float y;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ReanalyzeResult.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ReanalyzeResult {
-
-    private RedactionLog redactionLog;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Rectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-    private int page;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -1,37 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLog {
-
-    /**
-     * Version 0 Redaction Logs have manual redactions merged inside them
-     * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
-     */
-    private long analysisVersion;
-
-    /**
-     * Which analysis created this redactionLog.
-     */
-    private int analysisNumber;
-
-    private List<RedactionLogEntry> redactionLogEntry = new ArrayList<>();
-    private List<RedactionLogLegalBasis> legalBasis = new ArrayList<>();
-
-    private long dictionaryVersion = -1;
-    private long dossierDictionaryVersion = -1;
-    private long rulesVersion = -1;
-    private long legalBasisVersion = -1;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLogChanges {
-
-    private RedactionLog redactionLog;
-    private boolean hasChanges;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogComment {
-
-    private long id;
-    private String user;
-    private String text;
-    private String annotationId;
-    private String fileId;
-    private OffsetDateTime date;
-    private OffsetDateTime softDeletedTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -1,96 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-
-import lombok.*;
-
-import java.util.*;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode
-public class RedactionLogEntry {
-
-    private String id;
-    private String type;
-    private String value;
-    private String reason;
-    private int matchedRule;
-    private boolean rectangle;
-    private String legalBasis;
-
-    private boolean imported;
-
-    private boolean redacted;
-    private boolean isHint;
-    private boolean isRecommendation;
-    private boolean isFalsePositive;
-
-    private String section;
-    private float[] color;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-    private int sectionNumber;
-
-    private String textBefore;
-    private String textAfter;
-
-    @Builder.Default
-    private List<RedactionLogComment> comments = new ArrayList<>();
-
-    private int startOffset;
-    private int endOffset;
-
-    private boolean isImage;
-    private boolean imageHasTransparency;
-
-    private boolean isDictionaryEntry;
-    private boolean isDossierDictionaryEntry;
-
-    private boolean excluded;
-
-    private String sourceId;
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<Change> changes = new ArrayList<>();
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<ManualChange> manualChanges = new ArrayList<>();
-
-    private Set<Engine> engines = new HashSet<>();
-
-    private Set<String> reference = new HashSet<>();
-
-    @Builder.Default
-    private Set<String> importedRedactionIntersections = new HashSet<>();
-
-
-    public boolean lastChangeIsRemoved() {
-
-        return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
-    }
-
-
-    public boolean isLocalManualRedaction() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    public boolean isManuallyRemoved() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    private <T> Optional<T> last(List<T> list) {
-
-        return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogLegalBasis {
-
-    private String name;
-    private String description;
-    private String reason;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionRequest {
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-
-    private Colors colors;
-    private List<Type> types;
-
-    private boolean includeFalsePositives;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionResult {
-
-    private byte[] document;
-    private int numberOfPages;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class SectionArea {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int page;
-    private String header;
-
-
-    public boolean contains(Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft()
-                .getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft()
-                .getY() + other.getHeight();
-    }
-
-
-    // TODO we should only use one rectangle class.
-    public boolean contains(com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeftX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeftX() + other.getWidth() && this.getTopLeft()
-                .getY() <= other.getTopLeftY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeftY() + other.getHeight();
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java

@@ -1,33 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.*;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionGrid {
-
-    private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
-
-    private List<SectionGridSection> sections = new ArrayList<>();
-
-    @Data
-    @NoArgsConstructor
-    @AllArgsConstructor
-    public static class SectionGridSection {
-
-        private int sectionNumber;
-        private String headline;
-        private Set<Integer> pages;
-        private List<SectionArea> sectionAreas;
-
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int part;
-    private int numberOfParts;
-
-    private List<CellRectangle> tableCells;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -1,38 +1,12 @@
 package com.iqser.red.service.redaction.v1.resources;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 public interface RedactionResource {
 
-    @PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/sections", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult sections(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
-
-
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
     void testRules(@RequestBody String rules);
 
-
-    @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/manual/surrounding-text/{dossierId}/{fileId}", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions);
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -1,9 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -13,10 +9,17 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
-@Import({DefaultWebMvcConfiguration.class})
+@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 
 import lombok.Data;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 
-import java.util.ArrayList;
-import java.util.List;
-
 @Data
 @RequiredArgsConstructor
 public class Page {
@@ -32,7 +32,8 @@ public class Page {
     private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
 
-    private double cropBoxArea;
+    private float pageWidth;
+    private float pageHeight;
 
 
     public boolean isRotated() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -1,8 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -12,9 +16,6 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.ArrayList;
-import java.util.List;
-
 @AllArgsConstructor
 @Builder
 @Data
@@ -50,6 +51,139 @@ public class TextBlock extends AbstractTextContainer {
     private String classification;
 
 
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public TextDirection getDir() {
+
+        return sequences.get(0).getDir();
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    private float getPageHeight() {
+
+        return sequences.get(0).getPageHeight();
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    private float getPageWidth() {
+
+        return sequences.get(0).getPageWidth();
+    }
+
+
+    /**
+     * Returns the minX value in pdf coordinate system.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     *
+     * @return the minX value in pdf coordinate system
+     */
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getPdfMinX() {
+
+        if (getDir().getDegrees() == 90) {
+            return minY;
+        } else if (getDir().getDegrees() == 180) {
+            return getPageWidth() - maxX;
+
+        } else if (getDir().getDegrees() == 270) {
+
+            return getPageWidth() - maxY;
+        } else {
+            return minX;
+        }
+    }
+
+    /**
+     * Returns the maxX value in pdf coordinate system.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     *
+     * @return the maxX value in pdf coordinate system
+     */
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getPdfMaxX() {
+
+        if (getDir().getDegrees() == 90) {
+            return maxY;
+        } else if (getDir().getDegrees() == 180) {
+            return getPageWidth() - minX;
+        } else if (getDir().getDegrees() == 270) {
+            return getPageWidth() - minY;
+
+        } else {
+            return maxX;
+        }
+    }
+
+
+    /**
+     * Returns the minY value in pdf coordinate system.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     *
+     * @return the minY value in pdf coordinate system
+     */
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getPdfMinY() {
+
+        if (getDir().getDegrees() == 90) {
+            return minX;
+        } else if (getDir().getDegrees() == 180) {
+            return maxY;
+
+        } else if (getDir().getDegrees() == 270) {
+            return getPageHeight() - maxX;
+
+        } else {
+            return getPageHeight() - maxY;
+        }
+    }
+
+
+    /**
+     * Returns the maxY value in pdf coordinate system.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     *
+     * @return the maxY value in pdf coordinate system
+     */
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getPdfMaxY() {
+
+        if (getDir().getDegrees() == 90) {
+            return maxX;
+        } else if (getDir().getDegrees() == 180) {
+
+            return minY;
+        } else if (getDir().getDegrees() == 270) {
+            return getPageHeight() - minX;
+        } else {
+            return getPageHeight() - minY;
+        }
+    }
+
+
     public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
 
         this.minX = minX;
@@ -97,17 +231,17 @@ public class TextBlock extends AbstractTextContainer {
 
     public void add(TextPositionSequence r) {
 
-        if (r.getX1() < minX) {
-            minX = r.getX1();
+        if (r.getMinXDirAdj() < minX) {
+            minX = r.getMinXDirAdj();
         }
-        if (r.getX2() > maxX) {
-            maxX = r.getX2();
+        if (r.getMaxXDirAdj() > maxX) {
+            maxX = r.getMaxXDirAdj();
         }
-        if (r.getY1() < minY) {
-            minY = r.getY1();
+        if (r.getMinYDirAdj() < minY) {
+            minY = r.getMinYDirAdj();
         }
-        if (r.getY2() > maxY) {
-            maxY = r.getY2();
+        if (r.getMaxYDirAdj() > maxY) {
+            maxY = r.getMaxYDirAdj();
         }
     }
 
@@ -162,7 +296,7 @@ public class TextBlock extends AbstractTextContainer {
         TextPositionSequence previous = null;
         for (TextPositionSequence word : sequences) {
             if (previous != null) {
-                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                     sb.append('\n');
                 } else {
                     sb.append(' ');

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -14,13 +14,10 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Orientatio
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 @Service
 @SuppressWarnings("all")
@@ -29,10 +26,17 @@ public class BlockificationService {
     static final float THRESHOLD = 1f;
 
 
+    /**
+     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
+     * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
+     * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
+     * @param textPositions The words of a page.
+     * @param horizontalRulingLines Horizontal table lines.
+     * @param verticalRulingLines Vertical table lines.
+     * @return Page object that contains the Textblock and text statistics.
+     */
     public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
-        sortRotatedSequences(textPositions);
-
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
 
@@ -43,23 +47,15 @@ public class BlockificationService {
         Float splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
-            boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
-            boolean startFromTop = word.getY1() > maxY + word.getHeight();
-            boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
-            boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
-            boolean splittedByRuling = isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines) || isSplittedByRuling(minX,
-                    minY,
-                    word.getX1(),
-                    word.getY2(),
-                    horizontalRulingLines)
+            boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
+            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
+            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
+            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
+            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
-                    || isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || isSplittedByRuling(minX,
-                    minY,
-                    word.getX1(),
-                    word.getY2(),
-                    verticalRulingLines);
-
-            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -70,15 +66,15 @@ public class BlockificationService {
                 chunkBlockList1.add(cb1);
                 chunkWords = new ArrayList<>();
 
-                if (splitByX && !splittedByRuling) {
+                if (splitByX && !isSplitByRuling) {
                     wasSplitted = true;
                     cb1.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getX1();
-                } else if (newLineAfterSplit && !splittedByRuling) {
+                    splitX1 = word.getMinXDirAdj();
+                } else if (newLineAfterSplit && !isSplitByRuling) {
                     wasSplitted = false;
                     cb1.setOrientation(Orientation.RIGHT);
                     splitX1 = null;
-                } else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !splittedByRuling)) {
+                } else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !isSplitByRuling)) {
                     cb1.setOrientation(Orientation.LEFT);
                 }
 
@@ -92,17 +88,17 @@ public class BlockificationService {
             chunkWords.add(word);
 
             prev = word;
-            if (word.getX1() < minX) {
-                minX = word.getX1();
+            if (word.getMinXDirAdj() < minX) {
+                minX = word.getMinXDirAdj();
             }
-            if (word.getX2() > maxX) {
-                maxX = word.getX2();
+            if (word.getMaxXDirAdj() > maxX) {
+                maxX = word.getMaxXDirAdj();
             }
-            if (word.getY1() < minY) {
-                minY = word.getY1();
+            if (word.getMinYDirAdj() < minY) {
+                minY = word.getMinYDirAdj();
             }
-            if (word.getY2() > maxY) {
-                maxY = word.getY2();
+            if (word.getMaxYDirAdj() > maxY) {
+                maxY = word.getMaxYDirAdj();
             }
         }
 
@@ -186,7 +182,7 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
@@ -202,17 +198,33 @@ public class BlockificationService {
             textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
         }
 
-        if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) {
-            textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1));
+        if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getMinYDirAdj(), 3)).collect(toSet()).size() == 1) {
+            textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getMinXDirAdj));
         }
         return textBlock;
     }
 
 
-    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
+    private boolean isSplitByRuling(float minX,
+                                    float minY,
+                                    float maxX,
+                                    float maxY,
+                                    TextPositionSequence word,
+                                    List<Ruling> horizontalRulingLines,
+                                    List<Ruling> verticalRulingLines) {
+
+        return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                || isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); //
+    }
+
+
+    private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
 
         for (Ruling ruling : rulingLines) {
-            if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
+            var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
+            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
                 return true;
             }
         }
@@ -220,103 +232,6 @@ public class BlockificationService {
     }
 
 
-    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
-
-        float minX = 10000;
-        float maxX = -100;
-        float minY = 10000;
-        float maxY = -100;
-
-        for (Page page : pages) {
-
-            if (page.getTextBlocks().isEmpty() || landscape != page.isLandscape()) {
-                continue;
-            }
-
-            for (AbstractTextContainer container : page.getTextBlocks()) {
-
-                if (container instanceof TextBlock) {
-                    TextBlock textBlock = (TextBlock) container;
-                    if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
-                        continue;
-                    }
-
-                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < 2.9f) {
-                        continue;
-                    }
-
-                    if (documentFontSizeCounter.getMostPopular() != null) {
-                        if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
-
-                            if (textBlock.getMinX() < minX) {
-                                minX = textBlock.getMinX();
-                            }
-                            if (textBlock.getMaxX() > maxX) {
-                                maxX = textBlock.getMaxX();
-                            }
-                            if (textBlock.getMinY() < minY) {
-                                minY = textBlock.getMinY();
-                            }
-                            if (textBlock.getMaxY() > maxY) {
-                                maxY = textBlock.getMaxY();
-                            }
-
-                        }
-                    }
-                }
-
-                if (container instanceof Table) {
-                    Table table = (Table) container;
-                    for (List<Cell> row : table.getRows()) {
-                        for (Cell cell : row) {
-
-                            if (cell == null || cell.getTextBlocks() == null) {
-                                continue;
-                            }
-                            for (TextBlock textBlock : cell.getTextBlocks()) {
-                                if (textBlock.getMinX() < minX) {
-                                    minX = textBlock.getMinX();
-                                }
-                                if (textBlock.getMaxX() > maxX) {
-                                    maxX = textBlock.getMaxX();
-                                }
-                                if (textBlock.getMinY() < minY) {
-                                    minY = textBlock.getMinY();
-                                }
-                                if (textBlock.getMaxY() > maxY) {
-                                    maxY = textBlock.getMaxY();
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-        }
-        return new Rectangle(minY, minX, maxX - minX, maxY - minY);
-    }
-
-
-    private void sortRotatedSequences(List<TextPositionSequence> sequences) {
-
-        List<TextPositionSequence> rotatedWords = new ArrayList<>();
-        Iterator<TextPositionSequence> itty = sequences.iterator();
-        while (itty.hasNext()) {
-            var pos = itty.next();
-            if (pos.getTextPositions().get(0).getDir() == 270) {
-                rotatedWords.add(pos);
-                itty.remove();
-            }
-        }
-
-        if (!rotatedWords.isEmpty() && !sequences.isEmpty()) {
-            rotatedWords.sort(Comparator.comparing(TextPositionSequence::getX1));
-        }
-        sequences.addAll(rotatedWords);
-    }
-
-
     private double round(float value, int decimalPoints) {
 
         var d = Math.pow(10, decimalPoints);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -0,0 +1,161 @@
+package com.iqser.red.service.redaction.v1.server.classification.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
+@Service
+public class BodyTextFrameService {
+
+    /**
+     * Adjusts and sets the body text frame to a page.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     * The aspect ratio of the page is also regarded.
+     *
+     * @param page                   The page
+     * @param bodyTextFrame          frame that contains the main text on portrait pages
+     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
+     */
+    public void setBodyTextFrameAdjustedToPage(Page page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
+
+        Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
+
+        if (page.getPageWidth() > page.getPageHeight() && page.getRotation() == 270) {
+            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), page.getPageHeight() - textFrame.getTopLeft().getX() - textFrame.getWidth()),
+                    textFrame.getHeight(),
+                    textFrame.getWidth(),
+                    0);
+        } else if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
+            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), textFrame.getTopLeft().getX()), textFrame.getHeight(), textFrame.getWidth(), page.getPageNumber());
+        } else if (page.getRotation() == 180) {
+            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getX(), page.getPageHeight() - textFrame.getTopLeft().getY() - textFrame.getHeight()),
+                    textFrame.getWidth(),
+                    textFrame.getHeight(),
+                    0);
+        }
+        page.setBodyTextFrame(textFrame);
+    }
+
+
+    /**
+     * Calculates the frame that contains the main text, text outside the frame will be e.g. headers or footers.
+     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     * The aspect ratio of the page is also regarded.
+     *
+     * @param pages                   List of all pages
+     * @param documentFontSizeCounter Statistics of the document
+     * @param landscape               Calculate for landscape or portrait
+     * @return Rectangle of the text frame
+     */
+    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
+
+        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
+
+        for (Page page : pages) {
+
+            if (page.getTextBlocks().isEmpty() || landscape != page.isLandscape()) {
+                continue;
+            }
+
+            for (AbstractTextContainer container : page.getTextBlocks()) {
+
+                if (container instanceof TextBlock) {
+                    TextBlock textBlock = (TextBlock) container;
+                    if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
+                        continue;
+                    }
+
+                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
+                    if (approxLineCount < 2.9f) {
+                        continue;
+                    }
+
+                    if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
+
+                        expandRectangle(textBlock, page, expansionsRectangle);
+                    }
+                }
+
+                if (container instanceof Table) {
+                    Table table = (Table) container;
+                    for (List<Cell> row : table.getRows()) {
+                        for (Cell cell : row) {
+
+                            if (cell == null || cell.getTextBlocks() == null) {
+                                continue;
+                            }
+                            for (TextBlock textBlock : cell.getTextBlocks()) {
+                                expandRectangle(textBlock, page, expansionsRectangle);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return new Rectangle(new Point(expansionsRectangle.minX, expansionsRectangle.minY),
+                expansionsRectangle.maxX - expansionsRectangle.minX,
+                expansionsRectangle.maxY - expansionsRectangle.minY,
+                0);
+    }
+
+
+    private void expandRectangle(TextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
+
+        if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
+            if (textBlock.getPdfMinY() < expansionsRectangle.minX) {
+                expansionsRectangle.minX = textBlock.getPdfMinY();
+            }
+            if (textBlock.getPdfMaxY() > expansionsRectangle.maxX) {
+                expansionsRectangle.maxX = textBlock.getPdfMaxY();
+            }
+            if (textBlock.getPdfMinX() < expansionsRectangle.minY) {
+                expansionsRectangle.minY = textBlock.getPdfMinX();
+            }
+            if (textBlock.getPdfMaxX() > expansionsRectangle.maxY) {
+                expansionsRectangle.maxY = textBlock.getPdfMaxX();
+            }
+        } else {
+            if (textBlock.getPdfMinX() < expansionsRectangle.minX) {
+                expansionsRectangle.minX = textBlock.getPdfMinX();
+            }
+            if (textBlock.getPdfMaxX() > expansionsRectangle.maxX) {
+                expansionsRectangle.maxX = textBlock.getPdfMaxX();
+            }
+            if (textBlock.getPdfMinY() < expansionsRectangle.minY) {
+                expansionsRectangle.minY = textBlock.getPdfMinY();
+            }
+            if (textBlock.getPdfMaxY() > expansionsRectangle.maxY) {
+                expansionsRectangle.maxY = textBlock.getPdfMaxY();
+            }
+        }
+    }
+
+
+    private class BodyTextFrameExpansionsRectangle {
+
+        float minX = 10000;
+        float maxX = -100;
+        float minY = 10000;
+        float maxY = -100;
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,76 +1,75 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.regex.Pattern;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class ClassificationService {
 
-    private final BlockificationService blockificationService;
+    private final BodyTextFrameService bodyTextFrameService;
 
 
     public void classifyDocument(Document document) {
 
-        Rectangle bodyTextFrame = blockificationService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
-        Rectangle landscapeBodyTextFrame = blockificationService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
-
+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
         List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
         for (Page page : document.getPages()) {
-            Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
-            page.setBodyTextFrame(btf);
-            classifyPage(btf, page, document, headlineFontSizes);
+            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+            classifyPage(page, document, headlineFontSizes);
         }
     }
 
 
-    public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
+    public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) {
 
         for (AbstractTextContainer textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextBlock) {
-                classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
+                classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
+    public void classifyBlock(TextBlock textBlock, Page page, Document document, List<Float> headlineFontSizes) {
+
+        var bodyTextFrame = page.getBodyTextFrame();
 
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification("Other");
             return;
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Header");
 
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame,
-                textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
+        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                 document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
                 .size() == 1)) {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification("Title");
             }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
                 .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
                 .getCountPerValue()
                 .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java

@@ -1,25 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.classification.utils;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 @SuppressWarnings("all")
-public class PositionUtils {
+public final class PositionUtils {
 
+    // TODO This currently uses pdf coord system. In the futher this should use java coord system.
+    // Note: DirAdj (TextDirection Adjusted) can not be user for this.
     public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
 
-        //TODO Currently this is not working for rotated pages.
-
         if (btf == null || textBlock == null) {
             return false;
         }
 
         double threshold = textBlock.getMostPopularWordHeight() * 3;
 
-        if (textBlock.getMinX() + threshold > btf.getX() && textBlock.getMaxX() - threshold < btf.getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getY() && textBlock.getMaxY() - threshold < btf.getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
+                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
+                .getY() + btf.getHeight()) {
             return true;
         } else {
             return false;
@@ -28,16 +30,27 @@ public class PositionUtils {
     }
 
 
-    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, boolean rotated) {
+    // TODO This currently uses pdf coord system. In the futher this should use java coord system.
+    // Note: DirAdj (TextDirection Adjusted) can not be user for this.
+    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
         }
 
-        if (rotated && textBlock.getMinX() < btf.getX()) {
-            // Its very strange, P{0,0} is on top left in this case, instead of lower left.
+        if (rotation == 90 && textBlock.getPdfMaxX() < btf.getTopLeft().getX()) {
             return true;
-        } else if (!rotated && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
+        }
+
+        if (rotation == 180 && textBlock.getPdfMaxY() < btf.getTopLeft().getY()) {
+            return true;
+        }
+
+        if (rotation == 270 && textBlock.getPdfMinX() > btf.getTopLeft().getX() + btf.getWidth()) {
+            return true;
+        }
+
+        if (rotation == 0 && textBlock.getPdfMinY() > btf.getTopLeft().getY() + btf.getHeight()) {
             return true;
         } else {
             return false;
@@ -45,16 +58,27 @@ public class PositionUtils {
 
     }
 
-
-    public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
-
-        //TODO Currently this is not working for rotated pages.
+    // TODO This currently uses pdf coord system. In the futher this should use java coord system.
+    // Note: DirAdj (TextDirection Adjusted) can not be user for this.
+    public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
         }
 
-        if (textBlock.getMaxY() < btf.getY()) {
+        if (rotation == 90 && textBlock.getPdfMinX() > btf.getTopLeft().getX() + btf.getWidth()) {
+            return true;
+        }
+
+        if (rotation == 180 && textBlock.getPdfMinY() > btf.getTopLeft().getY() + btf.getHeight()) {
+            return true;
+        }
+
+        if (rotation == 270 && textBlock.getPdfMaxX() < btf.getTopLeft().getX()) {
+            return true;
+        }
+
+        if (rotation == 0 && textBlock.getPdfMaxY() < btf.getTopLeft().getY()) {
             return true;
         } else {
             return false;
@@ -62,7 +86,8 @@ public class PositionUtils {
 
     }
 
-
+    // TODO This currently uses pdf coord system. In the futher this should use java coord system.
+    // Note: DirAdj (TextDirection Adjusted) can not be user for this.
     public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
 
         //TODO Currently this is not working for rotated pages.
@@ -71,7 +96,7 @@ public class PositionUtils {
             return false;
         }
 
-        if (textBlock.getMinY() < btf.getY()) {
+        if (textBlock.getMinY() < btf.getTopLeft().getY()) {
             return true;
         } else {
             return false;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/RulingTextDirAdjustUtil.java

@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.classification.utils;
+
+import java.awt.geom.Line2D;
+import java.awt.geom.Point2D;
+
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public final class RulingTextDirAdjustUtil {
+
+    /**
+     * Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox.
+     * This will get the y position of the text, adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
+     *
+     * See org.apache.pdfbox.text.TextPosition
+     */
+    public Line2D.Float convertToDirAdj(Ruling ruling, float dir, float pageWidth, float pageHeight) {
+
+        return new Line2D.Float(convertPoint(ruling.x1, ruling.y1, dir, pageWidth, pageHeight), convertPoint(ruling.x2, ruling.y2, dir, pageWidth, pageHeight));
+    }
+
+
+    private Point2D convertPoint(float x, float y, float dir, float pageWidth, float pageHeight) {
+
+        var xAdj = getXRot(x, y, dir, pageWidth, pageHeight);
+        var yLowerLeftRot = getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
+        var yAdj = dir == 0 || dir == 180 ? pageHeight - yLowerLeftRot : pageWidth - yLowerLeftRot;
+        return new Point2D.Float(xAdj, yAdj);
+    }
+
+
+    @SuppressWarnings("SuspiciousNameCombination")
+    private float getXRot(float x, float y, float dir, float pageWidth, float pageHeight) {
+
+        if (dir == 0) {
+            return x;
+        } else if (dir == 90) {
+            return y;
+        } else if (dir == 180) {
+            return pageWidth - x;
+        } else if (dir == 270) {
+            return pageHeight - y;
+        }
+        return 0;
+    }
+
+
+    private float getYLowerLeftRot(float x, float y, float dir, float pageWidth, float pageHeight) {
+
+        if (dir == 0) {
+            return y;
+        } else if (dir == 90) {
+            return pageWidth - x;
+        } else if (dir == 180) {
+            return pageHeight - y;
+        } else if (dir == 270) {
+            return x;
+        }
+        return 0;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/DictionaryClient.java

@@ -2,9 +2,9 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
 
 @FeignClient(name = "DictionaryResource", url = "${persistence-service.url}")
 public interface DictionaryClient extends DictionaryResource {
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-
-@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}")
-public interface EntityRecognitionClient {
-
-    @PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    NerEntities findAuthors(EntityRecognitionRequest entityRecognitionRequest);
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
 
 @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.LegalBasisMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.LegalBasisMappingResource;
 
 @FeignClient(name = "LegalBasisMappingResource", url = "${persistence-service.url}")
 public interface LegalBasisClient extends LegalBasisMappingResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/RulesClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.RulesResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
 
 @FeignClient(name = "RulesResource", url = "${persistence-service.url}")
 public interface RulesClient extends RulesResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
+
+@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
+public interface TenantsClient extends TenantsResource {
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionRequest {
-
-    private List<EntityRecognitionSection> data;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java

@@ -1,21 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionResult {
-
-    @Builder.Default
-    private Map<Integer, List<EntityRecogintionEntity>> entities = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,31 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -35,100 +15,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
 
-    private final PdfVisualisationService pdfVisualisationService;
     private final DroolsExecutionService droolsExecutionService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionStorageService redactionStorageService;
-    private final RedactionLogMergeService redactionLogMergeService;
-    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
-
-
-    @Override
-    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
-
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
-
-        Document classifiedDoc;
-
-        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-
-        StringBuilder sb = new StringBuilder();
-        for (Page page : classifiedDoc.getPages()) {
-            for (AbstractTextContainer textContainer : page.getTextBlocks()) {
-                if (textContainer instanceof Table) {
-                    Table table = (Table) textContainer;
-                    sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
-                }
-            }
-        }
-
-        return RedactionResult.builder().document(sb.toString().getBytes()).build();
-
-    }
 
 
     @Override
@@ -141,30 +28,4 @@ public class RedactionController implements RedactionResource {
         }
     }
 
-
-    @Override
-    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
-
-        return redactionLogMergeService.provideRedactionLog(redactionRequest);
-    }
-
-
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
-
-        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-            document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
-        }
-
-    }
-
-
-    @Override
-    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {
-
-        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
-        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
-        return result.getManualRedactions();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.util.concurrent.Executor;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+@Configuration
+public class AsyncConfig extends AsyncConfigurerSupport {
+
+    @Override
+    public Executor getAsyncExecutor() {
+
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+
+        executor.setCorePoolSize(7);
+        executor.setMaxPoolSize(42);
+        executor.setQueueCapacity(11);
+        executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
+        executor.setTaskDecorator(new TenantAwareTaskDecorator());
+        executor.initialize();
+
+        return executor;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java

@@ -0,0 +1,105 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import java.security.spec.KeySpec;
+import java.util.Base64;
+
+import javax.annotation.PostConstruct;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import lombok.SneakyThrows;
+
+@Service
+public class EncryptionDecryptionService {
+
+    @Value("${redaction-service.crypto.key:redaction}")
+    private String key;
+
+    private SecretKey secretKey;
+    private byte[] iv;
+
+
+    @SneakyThrows
+    @PostConstruct
+    protected void postConstruct() {
+
+        SecureRandom secureRandom = new SecureRandom();
+        iv = new byte[12];
+        secureRandom.nextBytes(iv);
+        secretKey = generateSecretKey(key, iv);
+    }
+
+
+    @SneakyThrows
+    public String encrypt(String strToEncrypt) {
+
+        return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
+    }
+
+
+    @SneakyThrows
+    public String decrypt(String strToDecrypt) {
+
+        byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
+        return new String(decrypt(bytes), StandardCharsets.UTF_8);
+    }
+
+
+    @SneakyThrows
+    public byte[] encrypt(byte[] data) {
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
+        byte[] encryptedData = cipher.doFinal(data);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
+        byteBuffer.putInt(iv.length);
+        byteBuffer.put(iv);
+        byteBuffer.put(encryptedData);
+        return byteBuffer.array();
+    }
+
+
+    @SneakyThrows
+    public byte[] decrypt(byte[] encryptedData) {
+
+        ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
+        int noonceSize = byteBuffer.getInt();
+        if (noonceSize < 12 || noonceSize >= 16) {
+            throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
+        }
+        byte[] iv = new byte[noonceSize];
+        byteBuffer.get(iv);
+
+        SecretKey secretKey = generateSecretKey(key, iv);
+
+        byte[] cipherBytes = new byte[byteBuffer.remaining()];
+        byteBuffer.get(cipherBytes);
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
+        return cipher.doFinal(cipherBytes);
+    }
+
+
+    @SneakyThrows
+    public SecretKey generateSecretKey(String password, byte[] iv) {
+
+        KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
+        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
+        byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
+        return new SecretKeySpec(key, "AES");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+
+import feign.RequestInterceptor;
+import feign.RequestTemplate;
+
+@Component
+public class ForwardTenantInterceptor implements RequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+    @Override
+    public void apply(RequestTemplate template) {
+        template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+
+import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
+
+import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MultiTenancyMessagingConfiguration {
+
+    @Bean
+    public static BeanPostProcessor multitenancyBeanPostProcessor() {
+
+        return new BeanPostProcessor() {
+
+            @Override
+            public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
+
+                if (bean instanceof RabbitTemplate) {
+
+                    ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
+                        m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
+                        return m;
+                    });
+
+                } else if (bean instanceof AbstractRabbitListenerContainerFactory) {
+
+                    ((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
+                        String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
+
+                        if (tenant != null) {
+                            TenantContext.setTenantId(tenant);
+                        } else {
+                            throw new RuntimeException("No Tenant is set queue message");
+                        }
+                        return m;
+                    });
+                }
+                return bean;
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
+
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+
+@Configuration
+public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
+
+    private final TenantInterceptor tenantInterceptor;
+
+
+    @Autowired
+    public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
+
+        this.tenantInterceptor = tenantInterceptor;
+    }
+
+
+    @Override
+    public void addInterceptors(InterceptorRegistry registry) {
+
+        registry.addWebRequestInterceptor(tenantInterceptor);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java

@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.client.TenantsClient;
+import com.iqser.red.storage.commons.model.AzureStorageConnection;
+import com.iqser.red.storage.commons.model.S3StorageConnection;
+import com.iqser.red.storage.commons.service.StorageConnectionProvider;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class StorageConnectionProviderImpl implements StorageConnectionProvider {
+
+    private final TenantsClient tenantsClient;
+    private final EncryptionDecryptionService encryptionDecryptionService;
+
+
+    @Override
+    public AzureStorageConnection getAzureStorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return AzureStorageConnection.builder()
+                .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
+                .containerName(tenant.getAzureStorageConnection().getContainerName())
+                .build();
+    }
+
+
+    @Override
+    public S3StorageConnection getS3StorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return S3StorageConnection.builder()
+                .key(tenant.getS3StorageConnection().getKey())
+                .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
+                .signerType(tenant.getS3StorageConnection().getSignerType())
+                .bucketName(tenant.getS3StorageConnection().getBucketName())
+                .region(tenant.getS3StorageConnection().getRegion())
+                .endpoint(tenant.getS3StorageConnection().getEndpoint())
+                .build();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.core.task.TaskDecorator;
+import org.springframework.lang.NonNull;
+
+public class TenantAwareTaskDecorator implements TaskDecorator {
+
+    @Override
+    @NonNull
+    public Runnable decorate(@NonNull Runnable runnable) {
+
+        String tenantId = TenantContext.getTenantId();
+        return () -> {
+            try {
+                TenantContext.setTenantId(tenantId);
+                runnable.run();
+            } finally {
+                TenantContext.setTenantId(null);
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java

@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public final class TenantContext {
+
+    private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
+
+
+    public static void setTenantId(String tenantId) {
+
+        log.debug("Setting tenantId to " + tenantId);
+        currentTenant.set(tenantId);
+    }
+
+
+    public static String getTenantId() {
+
+        return currentTenant.get();
+    }
+
+
+    public static void clear() {
+
+        currentTenant.remove();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+import org.springframework.ui.ModelMap;
+import org.springframework.web.context.request.WebRequest;
+import org.springframework.web.context.request.WebRequestInterceptor;
+
+@Component
+public class TenantInterceptor implements WebRequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+
+    @Override
+    public void preHandle(WebRequest request) {
+
+        if (request.getHeader(TENANT_HEADER_NAME) != null) {
+            TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
+        }
+    }
+
+
+    @Override
+    public void postHandle(WebRequest request, ModelMap model) {
+
+        TenantContext.clear();
+    }
+
+
+    @Override
+    public void afterCompletion(WebRequest request, Exception ex) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java

@@ -22,6 +22,7 @@ import org.apache.pdfbox.text.TextPosition;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 
 @Slf4j
@@ -200,6 +201,8 @@ public class PDFLinesTextStripper extends PDFTextStripper {
         int startIndex = 0;
         RedTextPosition previous = null;
 
+        textPositions.sort(Comparator.comparing(TextPosition::getXDirAdj));
+
         for (int i = 0; i <= textPositions.size() - 1; i++) {
 
             if (!textPositionSequences.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -12,8 +12,8 @@ import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -47,16 +47,6 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
-
-        var textPositionSequence = new TextPositionSequence();
-        textPositionSequence.textPositions = textPositions;
-        textPositionSequence.page = page;
-
-        return textPositionSequence;
-    }
-
-
     public TextPositionSequence(List<TextPosition> textPositions, int page) {
 
         this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
@@ -147,59 +137,63 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
+    /**
+     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
+     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
+     *
+     * @return the text direction adjusted minX value
+     */
     @JsonIgnore
     @JsonAttribute(ignore = true)
-    public float getX1() {
+    public float getMinXDirAdj() {
+
+        return textPositions.get(0).getXDirAdj();
 
-        if (rotation == 90) {
-            return textPositions.get(0).getYDirAdj() - getTextHeight();
-        } else {
-            return textPositions.get(0).getXDirAdj();
-        }
     }
 
 
+    /**
+     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
+     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
+     *
+     * @return the text direction adjusted maxX value
+     */
     @JsonIgnore
     @JsonAttribute(ignore = true)
-    public float getX2() {
+    public float getMaxXDirAdj() {
+
+        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
 
-        if (rotation == 90) {
-            return textPositions.get(0).getYDirAdj();
-        } else {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
-        }
     }
 
 
+    /**
+     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
+     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
+     *
+     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
+     */
     @JsonIgnore
     @JsonAttribute(ignore = true)
-    public float getRotationAdjustedY() {
+    public float getMinYDirAdj() {
+
+        return textPositions.get(0).getYDirAdj() - getTextHeight();
+
+    }
+
+
+    /**
+     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
+     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
+     *
+     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
+     */
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getMaxYDirAdj() {
 
         return textPositions.get(0).getYDirAdj();
-    }
 
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getY1() {
-
-        if (rotation == 90) {
-            return textPositions.get(0).getXDirAdj();
-        } else {
-            return pageHeight - textPositions.get(0).getYDirAdj();
-        }
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getY2() {
-
-        if (rotation == 90) {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - HEIGHT_PADDING;
-        } else {
-            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
-        }
     }
 
 
@@ -215,7 +209,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getHeight() {
 
-        return getY2() - getY1();
+        return getMaxYDirAdj() - getMinYDirAdj();
     }
 
 
@@ -223,7 +217,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getWidth() {
 
-        return getX2() - getX1();
+        return getMaxXDirAdj() - getMinXDirAdj();
     }
 
 
@@ -270,6 +264,15 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
+    /**
+     * This returns the bounding box of the word in Pdf Coordinate System where {0,0} rotated with the page rotation.
+     * 0 -> LowerLeft
+     * 90 -> UpperLeft
+     * 180 -> UpperRight
+     * 270 -> LowerRight
+     *
+     * @return bounding box of the word in Pdf Coordinate System
+     */
     @JsonIgnore
     @JsonAttribute(ignore = true)
     @SneakyThrows

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -10,12 +10,12 @@ import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java

@@ -33,7 +33,7 @@ public class CellValue {
             TextPositionSequence previous = null;
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {
-                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');
                     } else {
                         sb.append(' ');

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java

@@ -5,6 +5,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import lombok.Data;
 import lombok.Getter;
 
@@ -49,8 +50,9 @@ public class Dictionary {
 
 
     public DictionaryModel getType(String type) {
-
-        return localAccessMap.get(type);
+        DictionaryModel model = localAccessMap.get(type);
+        if (model == null) throw new NotFoundException("Type: " + type + " is not found");
+        return model;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 import java.util.HashSet;
 import java.util.Set;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
 import lombok.*;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/FindEntitiesResult.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class FindEntitiesResult {
+
+    private Set<Entity> entities;
+    private Set<FileAttribute> addedFileAttributes;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/OffsetString.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class OffsetString {
+
+    private String value;
+    private int start;
+    private int end;
+
+
+    public OffsetString trim() {
+
+        String trimmed = this.value.trim();
+        int indexInUntrimmed = this.value.indexOf(trimmed);
+
+        int newStart = this.start + indexInUntrimmed;
+        int newEnd = newStart + trimmed.length();
+
+        return new OffsetString(trimmed, newStart, newEnd);
+    }
+
+}
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PageEntities.java

@@ -1,10 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -19,4 +22,7 @@ public class PageEntities {
     @Builder.Default
     private Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
 
+    @Builder.Default
+    private Set<FileAttribute> addedFileAttributes = new HashSet<>();
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -91,8 +91,8 @@ public class SearchableText {
 
                 if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i).charAt(j, caseInsensitive) == '-') {
 
-                    if (counter != 0 || i == 0 && j == 0 || j != 0 && SeparatorUtils.isSeparator(searchSpace.get(i)
-                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && (SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j - 1, caseInsensitive)) || SeparatorUtils.isSeparator(
+                            searchSpace.get(i).charAt(j, caseInsensitive))) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
                             .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                             .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i).charAt(j, caseInsensitive) != ' ') {
                         partMatch.add(searchSpace.get(i), searchSpace.get(i).textPositionAt(j));
@@ -105,9 +105,10 @@ public class SearchableText {
                         crossSequenceParts.add(partMatch);
 
                         if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
-                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1).charAt(0, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && searchSpace.get(i).charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1).charAt(0, caseInsensitive) != ' ') {
+                                .length() - 1 && (SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j + 1, caseInsensitive)) || SeparatorUtils.isSeparator(searchSpace.get(i)
+                                .charAt(j, caseInsensitive))) || j == searchSpace.get(i).length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1).charAt(0, caseInsensitive) != ' ') {
                             finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
                         }
 
@@ -118,7 +119,7 @@ public class SearchableText {
                 } else {
                     counter = 0;
                     if (!crossSequenceParts.isEmpty()) {
-                        j--;
+                        j = j - partMatch.length() - 1;
                     }
                     crossSequenceParts = new ArrayList<>();
                     partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
@@ -194,7 +195,7 @@ public class SearchableText {
         for (TextPositionSequence word : sequences) {
 
             if (previous != null) {
-                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                     sb.append('\n');
                 } else {
                     sb.append(' ');
@@ -228,7 +229,7 @@ public class SearchableText {
         for (TextPositionSequence word : sorted) {
 
             if (previous != null) {
-                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                     sb.append('\n');
                 } else {
                     sb.append(' ');
@@ -249,7 +250,7 @@ public class SearchableText {
         for (TextPositionSequence word : sequences) {
 
             if (previous != null) {
-                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                     sb.append('\n');
                 } else {
                     sb.append(' ');

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -9,7 +9,6 @@ import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -19,17 +18,18 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
@@ -75,45 +75,43 @@ public class Section {
     @Builder.Default
     private List<FileAttribute> fileAttributes = new ArrayList<>();
 
+    @Builder.Default
+    private Set<FileAttribute> addedFileAttributes = new HashSet<>();
+
     @Builder.Default
     private List<SectionArea> sectionAreas = new ArrayList<>();
 
     private ManualRedactions manualRedactions;
 
+    private boolean isInTable;
 
+
+    @Deprecated
     @SuppressWarnings("unused")
     @ThenAction
     public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
 
-        Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
-        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
-        Set<Entity> found = EntitySearchUtils.findEntities(searchText,
-                new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)),
-                dictionary.getType(asType),
-                new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
-        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
+        redactOrRecommendAiEntities(type, asType, false, 0, null, null);
+    }
 
-        Set<Entity> finalResult = new HashSet<>();
 
-        // Only keep Entities with correct offsets from AI Service.
-        for (Entity current : found) {
-            boolean foundSameOffsets = false;
-            for (Entity entity : entitiesOfType) {
-                if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
-                    foundSameOffsets = true;
-                    break;
-                }
-            }
-            if (foundSameOffsets) {
-                finalResult.add(current);
-            }
-        }
+    @SuppressWarnings("unused")
+    @ThenAction
+    public void recommendAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
 
-        var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, finalResult);
-        EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
-        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
-        nerEntities.removeAll(entitiesOfType);
+        redactOrRecommendAiEntities(type, asType, false, 0, null, null);
+    }
+
+
+    @SuppressWarnings("unused")
+    @ThenAction
+    public void redactAiEntities(@Argument(ArgumentType.TYPE) String type,
+                                 @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                 @Argument(ArgumentType.STRING) String reason,
+                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactOrRecommendAiEntities(type, asType, true, ruleNumber, reason, legalBasis);
     }
 
 
@@ -209,6 +207,12 @@ public class Section {
         return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
     }
 
+    @SuppressWarnings("unused")
+    @WhenCondition
+    public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) {
+
+        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue()));
+    }
 
     @SuppressWarnings("unused")
     @WhenCondition
@@ -235,6 +239,52 @@ public class Section {
     }
 
 
+    @SuppressWarnings("unused")
+    @ThenAction
+    public void addFileAttribute(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
+
+        if (fileAttributes == null) {
+            fileAttributes = new ArrayList<>();
+        }
+
+        boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel().equals(label) && f.getValue().equals(value));
+
+        if (!exists) {
+            fileAttributes.add(FileAttribute.builder().label(label).value(value).build());
+            addedFileAttributes.add(FileAttribute.builder().label(label).value(value).build());
+        }
+    }
+
+
+    @SuppressWarnings("unused")
+    @ThenAction
+    public void addFileAttribute(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
+                                 @Argument(ArgumentType.REGEX) String pattern,
+                                 @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                 @Argument(ArgumentType.INTEGER) int group) {
+
+        if (fileAttributes == null) {
+            fileAttributes = new ArrayList<>();
+        }
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(searchText);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                boolean exists = fileAttributes.stream().anyMatch(f -> f.getLabel().equals(label) && f.getValue().equals(match));
+
+                if (!exists) {
+                    fileAttributes.add(FileAttribute.builder().label(label).value(match).build());
+                    addedFileAttributes.add(FileAttribute.builder().label(label).value(match).build());
+                }
+            }
+        }
+    }
+
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
@@ -514,7 +564,36 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger);
     }
 
 
@@ -527,7 +606,7 @@ public class Section {
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, false);
     }
 
 
@@ -542,7 +621,7 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false, false);
     }
 
 
@@ -557,7 +636,7 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false, false);
     }
 
 
@@ -586,10 +665,43 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult);
+                sortedResult, false);
     }
 
 
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start,
+                              @Argument(ArgumentType.STRING) String stop,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStart,
+                              @Argument(ArgumentType.BOOLEAN) boolean includeStop,
+                              @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                              @Argument(ArgumentType.BOOLEAN) boolean ignoreTables) {
+
+        redactBetween(start,
+                stop,
+                includeStart,
+                includeStop,
+                asType,
+                ruleNumber,
+                redactEverywhere,
+                excludeHeadLine,
+                reason,
+                legalBasis,
+                true,
+                skipRemoveEntitiesContainedInLarger,
+                sortedResult, ignoreTables);
+    }
+
+
+
     @ThenAction
     @SuppressWarnings("unused")
     public void redactBetween(@Argument(ArgumentType.STRING) String start,
@@ -617,7 +729,7 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult);
+                sortedResult, false);
     }
 
 
@@ -660,7 +772,7 @@ public class Section {
                     legalBasis,
                     true,
                     skipRemoveEntitiesContainedInLarger,
-                    sortedResult);
+                    sortedResult, false);
         }
     }
 
@@ -675,7 +787,19 @@ public class Section {
                                  @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false);
+        redactBetween(start,
+                stop,
+                false,
+                false,
+                asType,
+                ruleNumber,
+                redactEverywhere,
+                false,
+                reason,
+                null,
+                false,
+                false,
+                false, false);
     }
 
 
@@ -689,7 +813,19 @@ public class Section {
                                  @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
                                  @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false);
+        redactBetween(start,
+                stop,
+                false,
+                false,
+                asType,
+                ruleNumber,
+                redactEverywhere,
+                excludeHeadLine,
+                reason,
+                null,
+                false,
+                false,
+                false, false);
     }
 
 
@@ -995,6 +1131,7 @@ public class Section {
 
 
     @ThenAction
+    @SuppressWarnings("unused")
     public void redactWordPartByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                                       @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                       @Argument(ArgumentType.INTEGER) int group,
@@ -1039,12 +1176,14 @@ public class Section {
                             }
                         }
 
-                        newSeq.setTextPositions(newPositions);
-                        entity.setWord(redactMatch);
-                        String plainId = IdBuilder.buildId(List.of(newSeq));
-                        entity.setPositionSequences(List.of(new EntityPositionSequence(plainId, List.of(newSeq), newSeq.getPage())));
+                        if (newSeq != null) {
+                            newSeq.setTextPositions(newPositions);
+                            entity.setWord(redactMatch);
+                            String plainId = IdBuilder.buildId(List.of(newSeq));
+                            entity.setPositionSequences(List.of(new EntityPositionSequence(plainId, List.of(newSeq), newSeq.getPage())));
 
-                        EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
+                            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
+                        }
                     }
                 }
             }
@@ -1089,6 +1228,48 @@ public class Section {
     }
 
 
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactLineAfterAcrossColumns(@Argument(ArgumentType.STRING) String start,
+                                             @Argument(ArgumentType.TYPE) String asType,
+                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                             @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                             @Argument(ArgumentType.STRING) String reason,
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, false, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactLineAfterAcrossColumns(@Argument(ArgumentType.STRING) String start,
+                                             @Argument(ArgumentType.TYPE) String asType,
+                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                             @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                             @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                                             @Argument(ArgumentType.STRING) String reason,
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactLineAfterAcrossColumns(@Argument(ArgumentType.STRING) String start,
+                                             @Argument(ArgumentType.TYPE) String asType,
+                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                             @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                             @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
+                                             @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch,
+                                             @Argument(ArgumentType.STRING) String reason,
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, onlyExactMatch);
+    }
+
+
     private void redactAndRecommendByRegEx(String pattern,
                                            boolean patternCaseInsensitive,
                                            int group,
@@ -1248,18 +1429,29 @@ public class Section {
     }
 
 
-    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis) {
+    private void redactLineAfterAcrossColumns(String start,
+                                              String asType,
+                                              int ruleNumber,
+                                              boolean redactEverywhere,
+                                              String reason,
+                                              String legalBasis,
+                                              boolean skipRemoveEntitiesContainedInLarger,
+                                              boolean onlyExactMatch) {
 
-        String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
+        var stringOffsets = OffsetStringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
+
+        if (stringOffsets != null) {
+            for (var stringOffset : stringOffsets) {
+                if (StringUtils.isNotBlank(stringOffset.getValue())) {
+                    var trimmedOffsetString = stringOffset.trim();
+                    Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
+                            .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet());
+                    found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
 
-        if (values != null) {
-            for (String value : values) {
-                if (StringUtils.isNotBlank(value)) {
-                    Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(trimmedOffsetString.getValue());
                     }
                 }
             }
@@ -1267,7 +1459,24 @@ public class Section {
     }
 
 
-    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) {
+    private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
+
+        Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                Set<Entity> found = findEntities(match.replaceAll("\\n", " ").trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+            }
+        }
+    }
+
+
+    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1277,6 +1486,7 @@ public class Section {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match)) {
                 Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
         }
@@ -1311,7 +1521,13 @@ public class Section {
                                String legalBasis,
                                boolean redaction,
                                boolean skipRemoveEntitiesContainedInLarger,
-                               boolean sortedResult) {
+                               boolean sortedResult,
+                               boolean ignoreTables) {
+
+
+        if(isInTable && ignoreTables){
+            return;
+        }
 
         String[] values = new String[1];
 
@@ -1394,6 +1610,51 @@ public class Section {
     }
 
 
+    private void redactOrRecommendAiEntities(String type, String asType, boolean redact, int ruleNumber, String reason, String legalBasis) {
+
+        Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
+        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
+        Set<Entity> found = EntitySearchUtils.findEntities(searchText,
+                new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)),
+                dictionary.getType(asType),
+                new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
+        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
+
+        Set<Entity> finalResult = new HashSet<>();
+
+        // Only keep Entities with correct offsets from AI Service.
+        for (Entity current : found) {
+            boolean foundSameOffsets = false;
+            for (Entity entity : entitiesOfType) {
+                if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
+                    foundSameOffsets = true;
+                    break;
+                }
+            }
+            if (foundSameOffsets) {
+                finalResult.add(current);
+            }
+        }
+
+        var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, finalResult);
+
+        if (redact) {
+            nonOverlapping.forEach(entity -> {
+                entity.setRedaction(true);
+                entity.setEntityType(EntityType.ENTITY);
+                entity.setMatchedRule(ruleNumber);
+                entity.setRedactionReason(reason);
+                entity.setLegalBasis(legalBasis);
+            });
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+        nerEntities.removeAll(entitiesOfType);
+    }
+
+
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.METHOD)
     public @interface WhenCondition {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -5,11 +5,11 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -51,8 +51,8 @@ public class ImportedRedactionService {
                                                                              List<RedactionLogEntry> redactionLogEntries,
                                                                              ImportedRedactions importedRedactions) {
 
-        for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (ImportedRedaction importedRedaction : importedRedactionsValues) {
+        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (var importedRedaction : importedRedactionsValues) {
                 RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
                         .id(importedRedaction.getId())
                         .type(IMPORTED_REDACTION_TYPE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -7,12 +7,11 @@ import java.util.Set;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -23,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
@@ -37,7 +37,7 @@ public class ManualRedactionSurroundingTextService {
     private final SurroundingWordsService surroundingWordsService;
 
 
-    @Timed("redactmanager_SurroundingTextAnalysis")
+    @Timed("redactmanager_surroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
 
         long startTime = System.currentTimeMillis();
@@ -125,7 +125,7 @@ public class ManualRedactionSurroundingTextService {
     }
 
 
-    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.redaction.v1.model.Rectangle textPositionRectangle) {
+    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle textPositionRectangle) {
 
         return textPositionRectangle.getTopLeft().getX() + textPositionRectangle.getWidth() > manualPosition.getTopLeftX() && textPositionRectangle.getTopLeft()
                 .getY() + textPositionRectangle.getHeight() > manualPosition.getTopLeftY() && textPositionRectangle.getTopLeft()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -9,9 +9,9 @@ import java.util.Set;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -1,544 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import java.awt.Color;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.Change;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class RedactionLogMergeService {
-
-    private static final String DELETED_TYPE_COLOR = "#9398a0";
-    private final SectionTextService sectionTextService;
-
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Timed("redactmanager_getMergedRedactionLog")
-    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
-
-        log.debug("Requested preview for: {}", redactionRequest);
-
-        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
-        if (sectionGrid.getSections().isEmpty()) {
-
-            log.info("SectionGrid does not have headlines set. Computing headlines now!");
-            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-            // enhance section grid with headline data
-            for (var sectionText : text.getSectionTexts()) {
-                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                                sectionText.getHeadline(),
-                                sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                                sectionText.getSectionAreas()));
-            }
-            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
-        }
-
-        log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = mergeRedactionLogData(redactionLog,
-                sectionGrid,
-                redactionRequest.getManualRedactions(),
-                redactionRequest.getExcludedPages(),
-                redactionRequest.getTypes(),
-                redactionRequest.getColors());
-
-        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
-
-        return merged;
-    }
-
-
-    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog,
-                                               SectionGrid sectionGrid,
-                                               ManualRedactions manualRedactions,
-                                               Set<Integer> excludedPages,
-                                               List<Type> types,
-                                               Colors colors) {
-
-        var skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
-        if (manualRedactions != null) {
-
-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid,
-                    manualRedactions.getEntriesToAdd(),
-                    manualRedactions.getComments(),
-                    colors,
-                    types,
-                    redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, types, colors);
-
-                if (entry.isImported() && !entry.isRedacted()) {
-                    skippedImportedRedactions.add(entry.getId());
-                }
-
-                entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));
-
-                if (excludedPages != null && !excludedPages.isEmpty()) {
-                    entry.getPositions().forEach(pos -> {
-                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
-                            entry.setExcluded(true);
-                        }
-                    });
-                }
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getImageRecategorization().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getIdsToRemove().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getForceRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getLegalBasisChanges().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getResizeRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            if (mrw.getItem() instanceof ManualImageRecategorization) {
-                var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
-                String manualOverrideReason = null;
-                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-                    redactionLogEntry.setType(imageRecategorization.getType());
-                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-                    if (isHint(types, imageRecategorization.getType())) {
-                        redactionLogEntry.setRedacted(false);
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setHint(false);
-                    }
-
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(imageRecategorization)
-                                .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                                .withChange("type", imageRecategorization.getType()));
-            }
-
-            if (mrw.getItem() instanceof IdRemoval) {
-                var manualRemoval = (IdRemoval) mrw.getItem();
-                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
-                    log.debug("Skipping merge for dictionary-modifying entry");
-                } else {
-                    String manualOverrideReason = null;
-                    if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
-                        redactionLogEntry.setRedacted(false);
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
-                        redactionLogEntry.setHint(false);
-                    } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    }
-
-                    if (manualOverrideReason != null) {
-                        redactionLogEntry.setReason(manualOverrideReason);
-                    }
-
-                    redactionLogEntry.getManualChanges()
-                            .add(ManualChange.from(manualRemoval)
-                                    .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-                }
-            }
-
-            if (mrw.getItem() instanceof ManualForceRedaction) {
-                var manualForceRedact = (ManualForceRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
-                if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    // Forcing a skipped hint should result in a hint
-                    if (dictionaryIsHint) {
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setRedacted(true);
-                    }
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualForceRedact)
-                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-                redactionLogEntry.getManualChanges().add(manualChange);
-
-            }
-
-            if (mrw.getItem() instanceof ManualLegalBasisChange) {
-                var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-                    redactionLogEntry.setRedacted(true);
-                    if (manualLegalBasisChange.getSection() != null) {
-                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-                    }
-                    if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                        redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-                    }
-                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-                manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-                if (manualLegalBasisChange.getSection() != null) {
-                    manualChange.withChange("section", manualLegalBasisChange.getSection());
-                }
-                if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                    manualChange.withChange("value", manualLegalBasisChange.getValue());
-                }
-                redactionLogEntry.getManualChanges().add(manualChange);
-            }
-
-            if (mrw.getItem() instanceof ManualResizeRedaction) {
-                var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-                    redactionLogEntry.setValue(manualResizeRedact.getValue());
-
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                }
-
-                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-            }
-
-        });
-
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
-                                                       Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments,
-                                                       Colors colors,
-                                                       List<Type> types,
-                                                       int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<RedactionLogComment> convert(List<Comment> comments) {
-
-        return comments == null ? null : comments.stream()
-                .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
-                .collect(Collectors.toList());
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, Colors colors, List<Type> types, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
-
-        if (requested) {
-            return convertColor(colors.getRequestRemoveColor());
-        }
-        if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
-
-        if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAddColor());
-        } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColor(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return convertColor(foundAndNotDeletedType.get().getHexColor());
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
-    }
-
-
-    boolean isHint(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return foundAndNotDeletedType.get().isHint();
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(Type::isHint).orElse(false);
-    }
-
-
-    private float[] convertColor(String hex) {
-
-        Color color = Color.decode(hex);
-        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
-    }
-
-
-    private List<Type> getMatchingTypes(List<Type> types, String type) {
-
-        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
-    }
-
-
-    private boolean isDeletedType(Type type) {
-
-        return type.getSoftDeletedTime() != null;
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}
-

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@@ -43,12 +43,13 @@ public class SectionGridCreatorService {
 
                 if (textBlock instanceof TextBlock) {
 
+                    TextBlock tb = (TextBlock) textBlock;
                     classifiedDoc.getSectionGrid()
                             .getRectanglesPerPage()
                             .computeIfAbsent(page, (x) -> new ArrayList<>())
-                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()),
-                                    textBlock.getWidth(),
-                                    textBlock.getHeight(),
+                            .add(new SectionRectangle(new Point(tb.getPdfMinX(), tb.getPdfMinY()),
+                                    tb.getPdfMaxX() - tb.getPdfMinX(),
+                                    tb.getPdfMaxY() - tb.getPdfMinY(),
                                     i + 1,
                                     paragraph.getPageBlocks().size(),
                                     null));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -11,8 +11,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java

@@ -1,41 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-public class SectionTextService {
-
-    public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getSection() != null) {
-            // set by UI
-            return;
-        }
-
-        if (sectionGrid != null) {
-            var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
-
-            if (firstPosition != null) {
-
-                for (var section : sectionGrid.getSections()) {
-                    if (section.getPages().contains(firstPosition.getPage())) {
-                        for (var sectionArea : section.getSectionAreas()) {
-                            if (sectionArea.contains(firstPosition)) {
-                                redactionLogEntry.setSection(section.getHeadline());
-                                redactionLogEntry.setSectionNumber(section.getSectionNumber());
-                                return;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -43,7 +43,7 @@ public class SurroundingWordsService {
     }
 
 
-    @Timed("redactmanager_addSurroundingText_tables")
+    @Timed("redactmanager_addSurroundingTextTables")
     public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
 
         if (entities.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -1,37 +1,30 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -43,42 +36,94 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionChangeLogService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionGridCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextBuilderService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction.EntityRedactionService;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
+import io.micrometer.core.instrument.FunctionTimer;
+import io.micrometer.core.instrument.MeterRegistry;
+import lombok.AccessLevel;
+import lombok.Getter;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class AnalyzeService {
 
-    private final DictionaryService dictionaryService;
-    private final DroolsExecutionService droolsExecutionService;
-    private final EntityRedactionService entityRedactionService;
-    private final RedactionLogCreatorService redactionLogCreatorService;
-    private final RedactionStorageService redactionStorageService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionChangeLogService redactionChangeLogService;
-    private final LegalBasisClient legalBasisClient;
-    private final RedactionServiceSettings redactionServiceSettings;
-    private final SectionTextBuilderService sectionTextBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final ImageService imageService;
-    private final ImportedRedactionService importedRedactionService;
+    DictionaryService dictionaryService;
+    DroolsExecutionService droolsExecutionService;
+    EntityRedactionService entityRedactionService;
+    RedactionLogCreatorService redactionLogCreatorService;
+    RedactionStorageService redactionStorageService;
+    PdfSegmentationService pdfSegmentationService;
+    RedactionChangeLogService redactionChangeLogService;
+    LegalBasisClient legalBasisClient;
+    RedactionServiceSettings redactionServiceSettings;
+    SectionTextBuilderService sectionTextBuilderService;
+    SectionGridCreatorService sectionGridCreatorService;
+    ImageService imageService;
+    ImportedRedactionService importedRedactionService;
+    SectionFinder sectionFinder;
+    PagewiseCounter analyzeCounter = new PagewiseCounter();
+
+
+    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
+    public AnalyzeService(DictionaryService dictionaryService,
+                          DroolsExecutionService droolsExecutionService,
+                          EntityRedactionService entityRedactionService,
+                          RedactionLogCreatorService redactionLogCreatorService,
+                          RedactionStorageService redactionStorageService,
+                          PdfSegmentationService pdfSegmentationService,
+                          RedactionChangeLogService redactionChangeLogService,
+                          LegalBasisClient legalBasisClient,
+                          RedactionServiceSettings redactionServiceSettings,
+                          SectionTextBuilderService sectionTextBuilderService,
+                          SectionGridCreatorService sectionGridCreatorService,
+                          ImageService imageService,
+                          ImportedRedactionService importedRedactionService,
+                          SectionFinder sectionFinder,
+                          MeterRegistry meterRegistry) {
+
+        this.dictionaryService = dictionaryService;
+        this.droolsExecutionService = droolsExecutionService;
+        this.entityRedactionService = entityRedactionService;
+        this.redactionLogCreatorService = redactionLogCreatorService;
+        this.redactionStorageService = redactionStorageService;
+        this.pdfSegmentationService = pdfSegmentationService;
+        this.redactionChangeLogService = redactionChangeLogService;
+        this.legalBasisClient = legalBasisClient;
+        this.redactionServiceSettings = redactionServiceSettings;
+        this.sectionTextBuilderService = sectionTextBuilderService;
+        this.sectionGridCreatorService = sectionGridCreatorService;
+        this.imageService = imageService;
+        this.importedRedactionService = importedRedactionService;
+        this.sectionFinder = sectionFinder;
+
+        // This is just here as an example of how to do a timer with a counter.
+        // For a full implementation this should be moved to a common helper class.
+        FunctionTimer.builder("redactmanager_analyze.pagewise", //
+                        analyzeCounter, //
+                        PagewiseCounter::getTotalPageCount, //
+                        PagewiseCounter::getTotalTimeMillis, //
+                        TimeUnit.MILLISECONDS) //
+                .register(meterRegistry);
+    }
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -152,32 +197,25 @@ public class AnalyzeService {
                 new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                 analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(
-                dictionaryIncrement,
-                redactionLog,
-                text,
-                analyzeRequest);
+        Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
 
         if (sectionsToReanalyse.isEmpty()) {
-            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
-        }
-
-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        } else {
-            nerEntities = new NerEntities();
+            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
         }
 
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         List<SectionText> reanalysisSections = text.getSectionTexts()
                 .stream()
                 .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                 .collect(Collectors.toList());
-
+        NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
+                ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
+                : new NerEntities();
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
 
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
         PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
 
         var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
@@ -191,7 +229,7 @@ public class AnalyzeService {
         redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
         redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
 
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, pageEntities.getAddedFileAttributes());
     }
 
 
@@ -233,50 +271,17 @@ public class AnalyzeService {
                 true);
         redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
 
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false, pageEntities.getAddedFileAttributes());
     }
 
 
-    @Timed("redactmanager_findSectionsToReanalyse")
-    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
-
-        long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
-
-        Set<Integer> sectionsToReanalyse = new HashSet<>();
-        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
-            }
-            if (entry.isImage()) {
-                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
-            }
-        }
-
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
-
-        for (SectionText sectionText : text.getSectionTexts()) {
-
-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
-                sectionsToReanalyse.add(sectionText.getSectionNumber());
-            }
-
-        }
-
-        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);
-
-        return sectionsToReanalyse;
-    }
-
-
-    private AnalyzeResult finalizeAnalysis(@RequestBody AnalyzeRequest analyzeRequest,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
                                            RedactionLog redactionLog,
                                            Text text,
                                            DictionaryVersion dictionaryVersion,
-                                           boolean isReanalysis) {
+                                           boolean isReanalysis,
+                                           Set<FileAttribute> addedFileAttributes) {
 
         redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
@@ -291,6 +296,8 @@ public class AnalyzeService {
 
         long duration = System.currentTimeMillis() - startTime;
 
+        analyzeCounter.increase(text.getNumberOfPages(), duration);
+
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
                 .fileId(analyzeRequest.getFileId())
@@ -305,45 +312,17 @@ public class AnalyzeService {
                 .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
                 .wasReanalyzed(isReanalysis)
                 .manualRedactions(analyzeRequest.getManualRedactions())
+                .addedFileAttributes(addedFileAttributes)
                 .build();
     }
 
 
-    private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return new HashSet<>();
-        }
-
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
-    }
-
-
     public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
     }
 
 
-    public Image convert(RedactionLogEntry entry) {
-
-        Rectangle position = entry.getPositions().get(0);
-
-        return Image.builder()
-                .type(entry.getType())
-                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
-                .sectionNumber(entry.getSectionNumber())
-                .section(entry.getSection())
-                .page(position.getPage())
-                .hasTransparency(entry.isImageHasTransparency())
-                .build();
-    }
-
-
     private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {
 
         if (excludedPages != null && !excludedPages.isEmpty()) {
@@ -368,4 +347,21 @@ public class AnalyzeService {
         return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
     }
 
+
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    @Getter
+    private static final class PagewiseCounter {
+
+        long totalPageCount;
+        long totalTimeMillis;
+
+
+        public void increase(long pageCount, long durationMillis) {
+
+            totalPageCount += pageCount;
+            totalTimeMillis += durationMillis;
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -0,0 +1,101 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class SectionFinder {
+    @Timed("redactmanager_findSectionsToReanalyse")
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
+
+        long start = System.currentTimeMillis();
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyse = new HashSet<>();
+        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
+        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+                sectionsToReanalyse.add(entry.getSectionNumber());
+            }
+            if (entry.isImage()) {
+                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
+            }
+        }
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
+                true);
+
+        for (SectionText sectionText : text.getSectionTexts()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
+                sectionsToReanalyse.add(sectionText.getSectionNumber());
+            }
+
+        }
+
+        log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
+
+        return sectionsToReanalyse;
+    }
+
+    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return new HashSet<>();
+        }
+
+        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
+                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
+                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
+                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
+                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+    }
+
+    private static Image convert(RedactionLogEntry entry) {
+
+        Rectangle position = entry.getPositions().get(0);
+
+        return Image.builder()
+                .type(entry.getType())
+                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
+                .sectionNumber(entry.getSectionNumber())
+                .section(entry.getSection())
+                .page(position.getPage())
+                .hasTransparency(entry.isImageHasTransparency())
+                .build();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java

@@ -0,0 +1,127 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class EntityFinder {
+
+    RedactionServiceSettings redactionServiceSettings;
+
+
+    @Timed("redactmanager_findEntities")
+    public Entities findEntities(SearchableText searchableText,
+                                 String headline,
+                                 int sectionNumber,
+                                 Dictionary dictionary,
+                                 boolean local,
+                                 NerEntities nerEntities,
+                                 List<Integer> cellStarts,
+                                 ManualRedactions manualRedactions) {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            return new Entities(new HashSet<>(), new HashSet<>());
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !local,
+                            model.isDossierDictionary(),
+                            local ? Engine.RULE : Engine.DICTIONARY,
+                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        Set<Entity> nerFound = new HashSet<>();
+        if (!local) {
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
+        }
+
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
+    }
+
+
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
+
+        Set<Entity> entities = new HashSet<>();
+
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
+            nerEntities.getData().get(sectionNumber).forEach(res -> {
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(),
+                            res.getType(),
+                            res.getStartOffset(),
+                            res.getEndOffset(),
+                            headline,
+                            sectionNumber,
+                            false,
+                            false,
+                            Engine.NER,
+                            EntityType.RECOMMENDATION));
+                } else {
+                    boolean intersectsCellStart = false;
+                    for (Integer cellStart : cellStarts) {
+                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
+                            intersectsCellStart = true;
+                            break;
+                        }
+                    }
+                    if (!intersectsCellStart) {
+                        entities.add(new Entity(res.getValue(),
+                                res.getType(),
+                                res.getStartOffset(),
+                                res.getEndOffset(),
+                                headline,
+                                sectionNumber,
+                                false,
+                                false,
+                                Engine.NER,
+                                EntityType.RECOMMENDATION));
+                    }
+                }
+            });
+        }
+        return entities;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -1,74 +1,162 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.commons.lang3.StringUtils;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityRedactionService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-    private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
+    DroolsExecutionService droolsExecutionService;
+    SurroundingWordsService surroundingWordsService;
+    EntityFinder entityFinder;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
 
         Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
-        Set<Entity> entities = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
+        FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
 
-        if (dictionary.hasLocalEntries()) {
-            Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(entities, dictionary);
-            Set<Entity> foundByLocal = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
-            EntitySearchUtils.addEntitiesWithHigherRank(entities, foundByLocal, dictionary);
-            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+        if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+
+            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+                //AnalyzeRequest provides immutable list.
+                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
+                mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
+                mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes());
+                analyzeRequest.setFileAttributes(mergedFileAttributes);
+            }
+
+            Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
+                    dictionary,
+                    kieContainer,
+                    analyzeRequest,
+                    true,
+                    hintsPerSectionNumber,
+                    imagesPerPage,
+                    nerEntities);
+            EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
+            EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
         }
 
-        Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
+        Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
         EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
 
-        return new PageEntities(entitiesPerPage, imagesPerPage);
+        return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
     }
 
 
-    public Set<Entity> findEntities(List<SectionText> reanalysisSections,
-                                    Dictionary dictionary,
-                                    KieContainer kieContainer,
-                                    AnalyzeRequest analyzeRequest,
-                                    boolean local,
-                                    Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                    Map<Integer, Set<Image>> imagesPerPage,
-                                    NerEntities nerEntities) {
+    public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
+                                           Dictionary dictionary,
+                                           KieContainer kieContainer,
+                                           AnalyzeRequest analyzeRequest,
+                                           boolean local,
+                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                           Map<Integer, Set<Image>> imagesPerPage,
+                                           NerEntities nerEntities) {
 
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-        for (SectionText reanalysisSection : reanalysisSections) {
+        List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
+                dictionary,
+                analyzeRequest,
+                local,
+                hintsPerSectionNumber,
+                nerEntities);
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+        Set<FileAttribute> addedFileAttributes = new HashSet<>();
+        Set<Entity> entities = new HashSet<>();
+        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+
+            if (!addedFileAttributes.isEmpty()) {
+                //Section.Builder provides immutable list.
+                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
+                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(addedFileAttributes);
+                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+            }
+
+            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+
+            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
+
+            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+
+            var entriesWithoutSurroundingText = analysedSection.getEntities()
+                    .stream()
+                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
+                    .collect(Collectors.toSet());
+
+            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        sectionSearchableTextPair.getSearchableText(),
+                        dictionary,
+                        sectionSearchableTextPair.getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+            }
+
+            entities.addAll(analysedSection.getEntities());
+
+            if (!local) {
+                for (Image image : analysedSection.getImages()) {
+                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+                }
+                addLocalValuesToDictionary(analysedSection, dictionary);
+            }
+
+        });
+
+        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
+    }
+
+
+    private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
+                                                                       Dictionary dictionary,
+                                                                       AnalyzeRequest analyzeRequest,
+                                                                       boolean local,
+                                                                       Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                       NerEntities nerEntities) {
+
+        return reanalysisSections.stream().map(reanalysisSection -> {
+
+            Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
                     reanalysisSection.getHeadline(),
                     reanalysisSection.getSectionNumber(),
                     dictionary,
@@ -126,58 +214,36 @@ public class EntityRedactionService {
 
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
 
-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                    .isLocal(false)
-                    .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
-                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                    .searchText(reanalysisSection.getSearchableText().toString())
-                    .headline(reanalysisSection.getHeadline())
-                    .sectionNumber(reanalysisSection.getSectionNumber())
-                    .tabularData(reanalysisSection.getTabularData())
-                    .searchableText(reanalysisSection.getSearchableText())
-                    .dictionary(dictionary)
-                    .images(reanalysisSection.getImages())
-                    .sectionAreas(reanalysisSection.getSectionAreas())
-                    .fileAttributes(analyzeRequest.getFileAttributes())
-                    .manualRedactions(analyzeRequest.getManualRedactions())
-                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
+            return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
+        }).collect(Collectors.toList());
+    }
 
-        }
 
-        Set<Entity> entities = new HashSet<>();
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+    private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
+                                                                  AnalyzeRequest analyzeRequest,
+                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                  SectionText reanalysisSection,
+                                                                  Entities entities) {
 
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
-                    .stream()
-                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
-                    .collect(Collectors.toSet());
-
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
-                        dictionary,
-                        sectionSearchableTextPair.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
-            }
-
-            entities.addAll(analysedSection.getEntities());
-
-            if (!local) {
-                for (Image image : analysedSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-                addLocalValuesToDictionary(analysedSection, dictionary);
-            }
-
-        });
-
-        return entities;
+        return new SectionSearchableTextPair(Section.builder()
+                .isLocal(false)
+                .dictionaryTypes(dictionary.getTypes())
+                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
+                        hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
+                .nerEntities(entities.getNerEntities())
+                .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                .searchText(reanalysisSection.getSearchableText().toString())
+                .headline(reanalysisSection.getHeadline())
+                .sectionNumber(reanalysisSection.getSectionNumber())
+                .tabularData(reanalysisSection.getTabularData())
+                .searchableText(reanalysisSection.getSearchableText())
+                .dictionary(dictionary)
+                .images(reanalysisSection.getImages())
+                .sectionAreas(reanalysisSection.getSectionAreas())
+                .fileAttributes(analyzeRequest.getFileAttributes())
+                .manualRedactions(analyzeRequest.getManualRedactions())
+                .isInTable(reanalysisSection.isTable())
+                .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
     }
 
 
@@ -219,7 +285,7 @@ public class EntityRedactionService {
     private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {
 
         Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
-        entities.stream().forEach(entity -> {
+        entities.forEach(entity -> {
             if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                 hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
             }
@@ -244,93 +310,4 @@ public class EntityRedactionService {
         }));
     }
 
-
-    @Timed("redactmanager_findEntities")
-    private Entities findEntities(SearchableText searchableText,
-                                  String headline,
-                                  int sectionNumber,
-                                  Dictionary dictionary,
-                                  boolean local,
-                                  NerEntities nerEntities,
-                                  List<Integer> cellStarts,
-                                  ManualRedactions manualRedactions) {
-
-        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.asString();
-
-        if (StringUtils.isEmpty(searchableString)) {
-            return new Entities(new HashSet<>(), new HashSet<>());
-        }
-
-        String lowercaseInputString = searchableString.toLowerCase();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-
-            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
-            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
-                    searchImplementation,
-                    model,
-                    new FindEntityDetails(model.getType(),
-                            headline,
-                            sectionNumber,
-                            !local,
-                            model.isDossierDictionary(),
-                            local ? Engine.RULE : Engine.DICTIONARY,
-                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
-
-            EntitySearchUtils.addOrAddEngine(found, entities);
-        }
-
-        Set<Entity> nerFound = new HashSet<>();
-        if (!local) {
-            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
-        }
-
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
-    }
-
-
-    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
-
-        Set<Entity> entities = new HashSet<>();
-
-        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
-            nerEntities.getData().get(sectionNumber).forEach(res -> {
-                if (cellStarts == null || cellStarts.isEmpty()) {
-                    entities.add(new Entity(res.getValue(),
-                            res.getType(),
-                            res.getStartOffset(),
-                            res.getEndOffset(),
-                            headline,
-                            sectionNumber,
-                            false,
-                            false,
-                            Engine.NER,
-                            EntityType.RECOMMENDATION));
-                } else {
-                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellStarts) {
-                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
-                            intersectsCellStart = true;
-                            break;
-                        }
-                    }
-                    if (!intersectsCellStart) {
-                        entities.add(new Entity(res.getValue(),
-                                res.getType(),
-                                res.getStartOffset(),
-                                res.getEndOffset(),
-                                headline,
-                                sectionNumber,
-                                false,
-                                false,
-                                Engine.NER,
-                                EntityType.RECOMMENDATION));
-                    }
-                }
-            });
-        }
-        return entities;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,20 +1,33 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
-public class EntitySearchUtils {
+public final class EntitySearchUtils {
 
     public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
 
@@ -26,7 +39,7 @@ public class EntitySearchUtils {
 
         Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
         markFalsePositives(found, falsePositives);
-        found.removeIf(f -> f.isFalsePositive());
+        found.removeIf(Entity::isFalsePositive);
     }
 
 
@@ -58,8 +71,8 @@ public class EntitySearchUtils {
 
     private void validateAndAddEntity(Set<Entity> entities, FindEntityDetails findEntityDetails, String inputString, int startIndex, int stopIndex) {
 
-        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(inputString.charAt(
-                stopIndex)))) {
+        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1)) || SeparatorUtils.isSeparator(inputString.charAt(startIndex))) && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(
+                inputString.charAt(stopIndex)) || SeparatorUtils.isSeparator(inputString.charAt(stopIndex - 1)))) {
             entities.add(new Entity(inputString.substring(startIndex, stopIndex),
                     findEntityDetails.getType(),
                     startIndex,
@@ -292,9 +305,12 @@ public class EntitySearchUtils {
                 .get(0)
                 .getSequences()
                 .get(0)
-                .getX1() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getX2() && image.getPosition()
-                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getY1() && image.getPosition().getY() + image.getPosition()
-                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getY2();
+                .getMinXDirAdj() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences()
+                .get(0)
+                .getSequences()
+                .get(0)
+                .getMaxXDirAdj() && image.getPosition().getY() < entity.getPositionSequences().get(0).getSequences().get(0).getMinYDirAdj() && image.getPosition()
+                .getY() + image.getPosition().getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxYDirAdj();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/FindEntityDetails.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -7,11 +10,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 
 import lombok.experimental.UtilityClass;
 
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
 @UtilityClass
-public class IdBuilder {
+public final class IdBuilder {
 
     private final HashFunction hashFunction = Hashing.murmur3_128();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.OffsetString;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public final class OffsetStringUtils {
+
+    /**
+     * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
+     *
+     * @param str – the String containing the substrings, null returns null, empty returns empty
+     * @param open – the String identifying the start of the substring, empty returns null
+     * @param close – the String identifying the end of the substring, empty returns null
+     * @return a list of Strings with their offsets
+     */
+    public List<OffsetString> substringsBetween(final String str, final String open, final String close) {
+
+        if (str == null || StringUtils.isEmpty(open) || StringUtils.isEmpty(close)) {
+            return null;
+        }
+        final int strLen = str.length();
+        if (strLen == 0) {
+            return new ArrayList<>();
+        }
+        final int closeLen = close.length();
+        final int openLen = open.length();
+        final List<OffsetString> list = new ArrayList<>();
+        int pos = 0;
+        while (pos < strLen - closeLen) {
+            int start = str.indexOf(open, pos);
+            if (start < 0) {
+                break;
+            }
+            start += openLen;
+            final int end = str.indexOf(close, start);
+            if (end < 0) {
+                break;
+            }
+            list.add(new OffsetString(str.substring(start, end), start, end));
+            pos = end + closeLen;
+        }
+        if (list.isEmpty()) {
+            return null;
+        }
+        return list;
+    }
+
+}
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java

@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.experimental.UtilityClass;
-
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Pattern;
 
+import lombok.experimental.UtilityClass;
+
 @UtilityClass
-public class Patterns {
+public final class Patterns {
 
     public static Map<String, Pattern> patternCache = new HashMap<>();
 
@@ -26,4 +26,16 @@ public class Patterns {
         return compiledPattern;
     }
 
+
+    public Pattern getCompiledMultilinePattern(String pattern, boolean caseInsensitive) {
+
+        String patternKey = pattern + caseInsensitive;
+        if (patternCache.containsKey(patternKey)) {
+            return patternCache.get(patternKey);
+        }
+        Pattern compiledPattern = Pattern.compile(pattern, (caseInsensitive ? Pattern.CASE_INSENSITIVE : 0) | Pattern.MULTILINE);
+        patternCache.put(patternKey, compiledPattern);
+        return compiledPattern;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.experimental.UtilityClass;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -10,8 +8,10 @@ import java.nio.charset.StandardCharsets;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import lombok.experimental.UtilityClass;
+
 @UtilityClass
-public class ResourceLoader {
+public final class ResourceLoader {
 
     public Set<String> load(String classpathPath) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java

@@ -3,14 +3,29 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 import java.util.Set;
 import java.util.regex.Pattern;
 
-public class SeparatorUtils {
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
+@UtilityClass
+public final class SeparatorUtils {
+
+    private final static Pattern punctuationPattern = Pattern.compile("\\p{Punct}");
     private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E', '\u2039', '\u203A');
 
+    private final static Set<Integer> japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309, 65306);
+
 
     public static boolean isSeparator(char c) {
 
-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c);
+        return Character.isWhitespace(c) || punctuationPattern.matcher(String.valueOf(c)).matches() || quotes.contains(c) || isJapaneseSeparator(c);
+    }
+
+
+    public static boolean isJapaneseSeparator(char c) {
+
+        var intValue = (int) c;
+        return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
-public class TextNormalizationUtilities {
+public final class TextNormalizationUtilities {
 
     /**
      * Revert hyphenation due to line breaks.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -4,18 +4,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -36,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -58,25 +53,14 @@ public class PdfSegmentationService {
     public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {
 
         PDDocument pdDocument = null;
+        File tempFile = null;
         try {
-            //create tempFile
-            File tempFile;
-
-            if (SystemUtils.IS_OS_UNIX) {
-                FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
-                tempFile = Files.createTempFile("document", ".pdf", attr).toFile();
-            } else {
-                tempFile = Files.createTempFile("document", ".pdf").toFile();
-                tempFile.setReadable(true, true);
-                tempFile.setWritable(true, true);
-                tempFile.setExecutable(true, true);
-            }
-
             Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
-            if (redactionServiceSettings.isCvServiceEnabled()) {
+            if (redactionServiceSettings.isCvTableParsingEnabled()) {
                 pdfTableCells = tableService.convertTables(dossierId, fileId);
             }
 
+            tempFile = FileUtils.createTempFile("document", ".pdf");
             try (var fos = new FileOutputStream(tempFile)) {
                 IOUtils.copy(documentInputStream, fos);
 
@@ -89,45 +73,7 @@ public class PdfSegmentationService {
                 long pageCount = pdDocument.getNumberOfPages();
 
                 for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                    stripper.setPageNumber(pageNumber);
-                    stripper.setStartPage(pageNumber);
-                    stripper.setEndPage(pageNumber);
-                    stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
-
-                    PDRectangle pdr = pdPage.getMediaBox();
-                    boolean isLandscape = pdr.getWidth() > pdr.getHeight();
-
-                    int rotation = pdPage.getRotation();
-                    boolean isRotated = rotation != 0 && rotation != 360;
-
-                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                            stripper.getRulings(),
-                            stripper.getMinCharWidth(),
-                            stripper.getMaxCharHeight());
-                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                    PDRectangle cropbox = pdPage.getCropBox();
-                    float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
-                    page.setCropBoxArea(cropboxArea);
-
-                    page.setRotation(rotation);
-                    page.setLandscape(isLandscape || isRotated);
-                    page.setPageNumber(pageNumber);
-
-                    tableExtractionService.extractTables(cleanRulings, page);
-                    buildPageStatistics(page);
-                    increaseDocumentStatistics(page, document);
-
-                    if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-                        page.setImages(pdfImages.get(pageNumber));
-                        imageService.findOcr(page);
-                    }
-
-                    pages.add(page);
+                    processPage(pdfImages, pdDocument, pdfTableCells, document, pages, pageNumber);
                 }
 
                 document.setPages(pages);
@@ -136,22 +82,65 @@ public class PdfSegmentationService {
                 sectionsBuilderService.buildSections(document);
                 sectionsBuilderService.addImagesToSections(document);
 
-                IOUtils.close(pdDocument);
-
-                if (!tempFile.delete()) {
-                    log.warn("Could not delete tmp file");
-                }
-
                 return document;
             }
         } finally {
             if (pdDocument != null) {
                 pdDocument.close();
             }
+
+            FileUtils.deleteFile(tempFile);
         }
     }
 
 
+    private void processPage(Map<Integer, List<PdfImage>> pdfImages,
+                           PDDocument pdDocument,
+                           Map<Integer, List<PdfTableCell>> pdfTableCells,
+                           Document document,
+                           List<Page> pages,
+                           int pageNumber) throws IOException {
+
+        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+        stripper.setPageNumber(pageNumber);
+        stripper.setStartPage(pageNumber);
+        stripper.setEndPage(pageNumber);
+        stripper.setPdpage(pdPage);
+        stripper.getText(pdDocument);
+
+        PDRectangle pdr = pdPage.getMediaBox();
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+        PDRectangle cropbox = pdPage.getCropBox();
+        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                stripper.getRulings(),
+                stripper.getMinCharWidth(),
+                stripper.getMaxCharHeight());
+        Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+        page.setRotation(rotation);
+        page.setLandscape(isLandscape);
+        page.setPageNumber(pageNumber);
+        page.setPageWidth(cropbox.getWidth());
+        page.setPageHeight(cropbox.getHeight());
+
+        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+            page.setImages(pdfImages.get(pageNumber));
+            imageService.findOcr(page);
+        }
+
+        tableExtractionService.extractTables(cleanRulings, page);
+        buildPageStatistics(page);
+        increaseDocumentStatistics(page, document);
+
+        pages.add(page);
+    }
+
+
     private void increaseDocumentStatistics(Page page, Document document) {
 
         if (!page.isLandscape()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TableService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
@@ -60,4 +60,4 @@ public class TableService {
         return pdfTableCells;
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java

@@ -14,7 +14,7 @@ public class RedactionServiceSettings {
 
     private boolean enableImageClassification = true;
 
-    private boolean cvServiceEnabled = true;
+    private boolean cvTableParsingEnabled = true;
 
     private float maxImageCropboxRatio = 0.9f;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -4,13 +4,14 @@ import java.io.InputStream;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 
@@ -31,14 +32,14 @@ public class RedactionStorageService {
     @SneakyThrows
     public InputStream getStoredObject(String storageId) {
 
-        return storageService.getObject(storageId).getInputStream();
+        return storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream();
     }
 
 
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
+        storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
     }
 
 
@@ -46,7 +47,7 @@ public class RedactionStorageService {
     @Timed("redactmanager_storeObject")
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
 
-        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
+        storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
     }
 
 
@@ -54,7 +55,7 @@ public class RedactionStorageService {
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
@@ -66,7 +67,7 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -79,7 +80,7 @@ public class RedactionStorageService {
     public Text getText(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -91,7 +92,7 @@ public class RedactionStorageService {
     public NerEntities getNerEntities(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("NER Entities are not available.");
         }
@@ -102,7 +103,7 @@ public class RedactionStorageService {
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("Section Grid is not available.");
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -2,8 +2,9 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -34,6 +35,12 @@ public abstract class AbstractTextContainer {
     public abstract String getText();
 
 
+    public boolean containsBlock(TextBlock other) {
+
+        return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
+    }
+
+
     public boolean contains(AbstractTextContainer other) {
 
         return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -51,7 +51,7 @@ public class Cell extends Rectangle {
 
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {
-                    if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');
                     } else {
                         sb.append(' ');